@mdream/js 1.2.2 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/const.mjs +3 -2
- package/dist/_chunks/index.d.mts +2 -2
- package/dist/_chunks/minimal.d.mts +2 -2
- package/dist/_chunks/parse.mjs +112 -12
- package/dist/_chunks/plugins.mjs +18 -10
- package/dist/_chunks/resolve-plugins.mjs +104 -7
- package/dist/_chunks/src.mjs +3 -3
- package/dist/_chunks/types.d.mts +14 -1
- package/dist/cli.mjs +6 -3
- package/dist/index.d.mts +3 -3
- package/dist/index.mjs +3 -3
- package/dist/parse.d.mts +8 -1
- package/dist/parse.mjs +1 -1
- package/dist/plugins.d.mts +2 -2
- package/dist/plugins.mjs +1 -1
- package/dist/preset/minimal.d.mts +1 -1
- package/dist/splitter.d.mts +1 -1
- package/dist/splitter.mjs +2 -2
- package/package.json +1 -1
package/dist/_chunks/const.mjs
CHANGED
|
@@ -116,7 +116,8 @@ const TagIdMap = {
|
|
|
116
116
|
main: 104,
|
|
117
117
|
header: 105,
|
|
118
118
|
figcaption: 106,
|
|
119
|
-
caption: 107
|
|
119
|
+
caption: 107,
|
|
120
|
+
datalist: 108
|
|
120
121
|
};
|
|
121
122
|
Object.entries(TagIdMap).map(([name, id]) => [id, name]);
|
|
122
123
|
const NO_SPACING = [0, 0];
|
|
@@ -124,4 +125,4 @@ const DEFAULT_BLOCK_SPACING = [2, 2];
|
|
|
124
125
|
const BLOCKQUOTE_SPACING = [1, 1];
|
|
125
126
|
const LIST_ITEM_SPACING = [1, 0];
|
|
126
127
|
const TABLE_ROW_SPACING = [0, 1];
|
|
127
|
-
export {
|
|
128
|
+
export { BLOCKQUOTE_SPACING, DEFAULT_BLOCK_SPACING, ELEMENT_NODE, LIST_ITEM_SPACING, NO_SPACING, NodeEventEnter, NodeEventExit, TABLE_ROW_SPACING, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, TagIdMap };
|
package/dist/_chunks/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MdreamOptions, TransformPlugin } from "./types.mjs";
|
|
2
2
|
/**
|
|
3
3
|
* Create a plugin with type-safe hook definitions.
|
|
4
4
|
* All TransformPlugin fields are optional, so this is a typed identity function.
|
|
@@ -6,4 +6,4 @@ import { _ as TransformPlugin, c as MdreamOptions } from "./types.mjs";
|
|
|
6
6
|
declare function createPlugin(plugin: TransformPlugin): TransformPlugin;
|
|
7
7
|
declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
|
|
8
8
|
declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
9
|
-
export {
|
|
9
|
+
export { createPlugin, htmlToMarkdown, streamHtmlToMarkdown };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { EngineOptions } from "./types.mjs";
|
|
2
2
|
/**
|
|
3
3
|
* Creates a configurable minimal preset with advanced options.
|
|
4
4
|
* Returns declarative plugin config that works with both JS and Rust engines.
|
|
5
5
|
*/
|
|
6
6
|
declare function withMinimalPreset<T extends EngineOptions>(options?: T): T;
|
|
7
|
-
export { withMinimalPreset
|
|
7
|
+
export { withMinimalPreset };
|
package/dist/_chunks/parse.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { BLOCKQUOTE_SPACING, LIST_ITEM_SPACING, NO_SPACING, TABLE_ROW_SPACING, TagIdMap } from "./const.mjs";
|
|
2
2
|
function resolveUrl(url, origin) {
|
|
3
3
|
if (!url) return url;
|
|
4
4
|
if (url.startsWith("//")) return `https:${url}`;
|
|
@@ -155,9 +155,29 @@ const tagHandlers = {
|
|
|
155
155
|
},
|
|
156
156
|
spacing: BLOCKQUOTE_SPACING
|
|
157
157
|
},
|
|
158
|
+
[34]: {
|
|
159
|
+
enter: ({ node, state }) => {
|
|
160
|
+
state.preFencePending = true;
|
|
161
|
+
state.preOwnFence = false;
|
|
162
|
+
state.preFenceLang = getLanguageFromClass(node.attributes?.class);
|
|
163
|
+
},
|
|
164
|
+
exit: ({ node, state }) => {
|
|
165
|
+
const ownFence = state.preOwnFence;
|
|
166
|
+
state.preFencePending = false;
|
|
167
|
+
state.preOwnFence = false;
|
|
168
|
+
if (!ownFence) return;
|
|
169
|
+
if ((node.depthMap[25] || 0) > 0) {
|
|
170
|
+
const indent = state.listIndent;
|
|
171
|
+
return `\n${indent}\`\`\`\n\n${indent}`;
|
|
172
|
+
}
|
|
173
|
+
return `\n\`\`\``;
|
|
174
|
+
}
|
|
175
|
+
},
|
|
158
176
|
[23]: {
|
|
159
177
|
enter: ({ node, state }) => {
|
|
160
178
|
if ((node.depthMap[34] || 0) > 0) {
|
|
179
|
+
if (state.preOwnFence) return;
|
|
180
|
+
state.preFencePending = false;
|
|
161
181
|
const language = getLanguageFromClass(node.attributes?.class);
|
|
162
182
|
if ((node.depthMap[25] || 0) > 0) return `\n\n${state.listIndent}\`\`\`${language}\n`;
|
|
163
183
|
return `\`\`\`${language}\n`;
|
|
@@ -171,6 +191,7 @@ const tagHandlers = {
|
|
|
171
191
|
},
|
|
172
192
|
exit: ({ node, state }) => {
|
|
173
193
|
if ((node.depthMap[34] || 0) > 0) {
|
|
194
|
+
if (state.preOwnFence) return;
|
|
174
195
|
if ((node.depthMap[25] || 0) > 0) {
|
|
175
196
|
const indent = state.listIndent;
|
|
176
197
|
return `\n${indent}\`\`\`\n\n${indent}`;
|
|
@@ -449,7 +470,11 @@ const tagHandlers = {
|
|
|
449
470
|
[75]: { spacing: NO_SPACING },
|
|
450
471
|
[76]: { spacing: NO_SPACING },
|
|
451
472
|
[77]: { spacing: NO_SPACING },
|
|
452
|
-
[78]: {
|
|
473
|
+
[78]: {
|
|
474
|
+
isNonNesting: true,
|
|
475
|
+
excludesTextNodes: true,
|
|
476
|
+
spacing: NO_SPACING
|
|
477
|
+
},
|
|
453
478
|
[79]: {
|
|
454
479
|
enter: () => "",
|
|
455
480
|
exit: () => "",
|
|
@@ -489,6 +514,11 @@ const tagHandlers = {
|
|
|
489
514
|
excludesTextNodes: true,
|
|
490
515
|
spacing: NO_SPACING
|
|
491
516
|
},
|
|
517
|
+
[108]: {
|
|
518
|
+
isNonNesting: true,
|
|
519
|
+
excludesTextNodes: true,
|
|
520
|
+
spacing: NO_SPACING
|
|
521
|
+
},
|
|
492
522
|
[85]: {
|
|
493
523
|
isNonNesting: true,
|
|
494
524
|
spacing: NO_SPACING
|
|
@@ -964,7 +994,7 @@ function parseHtml(html, options = {}) {
|
|
|
964
994
|
return {
|
|
965
995
|
events,
|
|
966
996
|
remainingHtml: parseHtmlInternal(html, {
|
|
967
|
-
depthMap: new Uint8Array(
|
|
997
|
+
depthMap: new Uint8Array(109),
|
|
968
998
|
depth: 0,
|
|
969
999
|
resolvedPlugins: options.resolvedPlugins || []
|
|
970
1000
|
}, (event) => {
|
|
@@ -977,7 +1007,7 @@ function parseHtmlStream(html, state, onEvent) {
|
|
|
977
1007
|
}
|
|
978
1008
|
function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
979
1009
|
let textBuffer = "";
|
|
980
|
-
state.depthMap ??= new Uint8Array(
|
|
1010
|
+
state.depthMap ??= new Uint8Array(109);
|
|
981
1011
|
state.depth ??= 0;
|
|
982
1012
|
state.lastCharWasWhitespace ??= true;
|
|
983
1013
|
state.justClosedTag ??= false;
|
|
@@ -1014,7 +1044,7 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1014
1044
|
else if (currentCharCode === CLOSE_BRACKET_CHAR && state.depthMap[26]) textBuffer += "\\]";
|
|
1015
1045
|
else if (currentCharCode === GT_CHAR && state.depthMap[22]) textBuffer += "\\>";
|
|
1016
1046
|
else textBuffer += htmlChunk[i];
|
|
1017
|
-
if (state.
|
|
1047
|
+
if (state.inRawTextQuoteAware) {
|
|
1018
1048
|
if (!state.lastCharWasBackslash) {
|
|
1019
1049
|
if (currentCharCode === APOS_CHAR && !state.inDoubleQuote && !state.inBacktick) state.inSingleQuote = !state.inSingleQuote;
|
|
1020
1050
|
else if (currentCharCode === QUOTE_CHAR && !state.inSingleQuote && !state.inBacktick) state.inDoubleQuote = !state.inDoubleQuote;
|
|
@@ -1032,6 +1062,26 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1032
1062
|
}
|
|
1033
1063
|
const nextCharCode = htmlChunk.charCodeAt(i + 1);
|
|
1034
1064
|
if (nextCharCode === EXCLAMATION_CHAR) {
|
|
1065
|
+
if (htmlChunk.charCodeAt(i + 2) === OPEN_BRACKET_CHAR) {
|
|
1066
|
+
if (htmlChunk.startsWith("<![CDATA[", i)) {
|
|
1067
|
+
const end = htmlChunk.indexOf("]]>", i + 9);
|
|
1068
|
+
if (end === -1) {
|
|
1069
|
+
textBuffer += htmlChunk.substring(i);
|
|
1070
|
+
break;
|
|
1071
|
+
}
|
|
1072
|
+
if (textBuffer.length > 0) {
|
|
1073
|
+
processTextBuffer(textBuffer, state, handleEvent);
|
|
1074
|
+
textBuffer = "";
|
|
1075
|
+
}
|
|
1076
|
+
processCdataSection(htmlChunk.substring(i + 9, end), state, handleEvent);
|
|
1077
|
+
i = end + 3;
|
|
1078
|
+
continue;
|
|
1079
|
+
}
|
|
1080
|
+
if (chunkLength - i < 9 && "<![CDATA[".startsWith(htmlChunk.substring(i))) {
|
|
1081
|
+
textBuffer += htmlChunk.substring(i);
|
|
1082
|
+
break;
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1035
1085
|
if (textBuffer.length > 0) {
|
|
1036
1086
|
processTextBuffer(textBuffer, state, handleEvent);
|
|
1037
1087
|
textBuffer = "";
|
|
@@ -1044,7 +1094,7 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1044
1094
|
}
|
|
1045
1095
|
} else if (nextCharCode === SLASH_CHAR) {
|
|
1046
1096
|
if (state.currentNode?.tagHandler?.isNonNesting) {
|
|
1047
|
-
if (state.inSingleQuote || state.inDoubleQuote || state.inBacktick) {
|
|
1097
|
+
if (state.inRawTextQuoteAware && (state.inSingleQuote || state.inDoubleQuote || state.inBacktick)) {
|
|
1048
1098
|
textBuffer += htmlChunk[i];
|
|
1049
1099
|
i++;
|
|
1050
1100
|
continue;
|
|
@@ -1090,10 +1140,8 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1090
1140
|
const tagId = TagIdMap[tagName] ?? -1;
|
|
1091
1141
|
i2 = tagNameEnd;
|
|
1092
1142
|
if (state.currentNode?.tagHandler?.isNonNesting) {
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
continue;
|
|
1096
|
-
}
|
|
1143
|
+
textBuffer += htmlChunk[i++];
|
|
1144
|
+
continue;
|
|
1097
1145
|
}
|
|
1098
1146
|
if (!tagName) {
|
|
1099
1147
|
textBuffer += htmlChunk[i++];
|
|
@@ -1114,6 +1162,10 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1114
1162
|
}
|
|
1115
1163
|
}
|
|
1116
1164
|
}
|
|
1165
|
+
state.inSingleQuote = false;
|
|
1166
|
+
state.inDoubleQuote = false;
|
|
1167
|
+
state.inBacktick = false;
|
|
1168
|
+
state.lastCharWasBackslash = false;
|
|
1117
1169
|
return textBuffer;
|
|
1118
1170
|
}
|
|
1119
1171
|
function processTextBuffer(textBuffer, state, handleEvent) {
|
|
@@ -1121,7 +1173,30 @@ function processTextBuffer(textBuffer, state, handleEvent) {
|
|
|
1121
1173
|
const containsWhitespace = state.textBufferContainsWhitespace;
|
|
1122
1174
|
state.textBufferContainsNonWhitespace = false;
|
|
1123
1175
|
state.textBufferContainsWhitespace = false;
|
|
1124
|
-
if (!state.currentNode)
|
|
1176
|
+
if (!state.currentNode) {
|
|
1177
|
+
if (!containsNonWhitespace) return;
|
|
1178
|
+
let rootText = textBuffer;
|
|
1179
|
+
if (rootText.length === 0) return;
|
|
1180
|
+
if (state.hasEncodedHtmlEntity) {
|
|
1181
|
+
rootText = decodeHTMLEntities(String(rootText));
|
|
1182
|
+
state.hasEncodedHtmlEntity = false;
|
|
1183
|
+
}
|
|
1184
|
+
const rootTextNode = {
|
|
1185
|
+
type: 2,
|
|
1186
|
+
value: rootText,
|
|
1187
|
+
parent: null,
|
|
1188
|
+
index: 0,
|
|
1189
|
+
depth: state.depth,
|
|
1190
|
+
containsWhitespace,
|
|
1191
|
+
excludedFromMarkdown: false
|
|
1192
|
+
};
|
|
1193
|
+
handleEvent({
|
|
1194
|
+
type: 0,
|
|
1195
|
+
node: rootTextNode
|
|
1196
|
+
});
|
|
1197
|
+
state.lastTextNode = rootTextNode;
|
|
1198
|
+
return;
|
|
1199
|
+
}
|
|
1125
1200
|
const excludesTextNodes = state.currentNode?.tagHandler?.excludesTextNodes;
|
|
1126
1201
|
const inPreTag = (state.depthMap[34] || 0) > 0;
|
|
1127
1202
|
if (!inPreTag && !containsNonWhitespace && !state.currentNode.childTextNodeIndex) return;
|
|
@@ -1219,6 +1294,7 @@ function closeNode(node, state, handleEvent) {
|
|
|
1219
1294
|
state.inDoubleQuote = false;
|
|
1220
1295
|
state.inBacktick = false;
|
|
1221
1296
|
state.lastCharWasBackslash = false;
|
|
1297
|
+
state.inRawTextQuoteAware = false;
|
|
1222
1298
|
}
|
|
1223
1299
|
state.depth--;
|
|
1224
1300
|
handleEvent({
|
|
@@ -1270,6 +1346,29 @@ function processCommentOrDoctype(htmlChunk, position) {
|
|
|
1270
1346
|
};
|
|
1271
1347
|
}
|
|
1272
1348
|
}
|
|
1349
|
+
function processCdataSection(content, state, handleEvent) {
|
|
1350
|
+
if (!state.tagOverrideHandlers?.has("#cdata-section")) return;
|
|
1351
|
+
const open = processOpeningTag("#cdata-section", -1, ">", 0, state, handleEvent);
|
|
1352
|
+
if (!open.complete || open.selfClosing) return;
|
|
1353
|
+
const node = state.currentNode;
|
|
1354
|
+
if (content.length > 0 && !node.tagHandler?.excludesTextNodes) {
|
|
1355
|
+
const textNode = {
|
|
1356
|
+
type: 2,
|
|
1357
|
+
value: content,
|
|
1358
|
+
parent: node,
|
|
1359
|
+
index: node.currentWalkIndex++,
|
|
1360
|
+
depth: state.depth,
|
|
1361
|
+
containsWhitespace: false
|
|
1362
|
+
};
|
|
1363
|
+
for (const parent of traverseUpToFirstBlockNode(node)) parent.childTextNodeIndex = (parent.childTextNodeIndex || 0) + 1;
|
|
1364
|
+
handleEvent({
|
|
1365
|
+
type: 0,
|
|
1366
|
+
node: textNode
|
|
1367
|
+
});
|
|
1368
|
+
state.lastTextNode = textNode;
|
|
1369
|
+
}
|
|
1370
|
+
closeNode(state.currentNode ?? null, state, handleEvent);
|
|
1371
|
+
}
|
|
1273
1372
|
function processOpeningTag(tagName, tagId, htmlChunk, i, state, handleEvent) {
|
|
1274
1373
|
if (state.currentNode?.tagHandler?.isNonNesting) closeNode(state.currentNode, state, handleEvent);
|
|
1275
1374
|
const tagHandler = state.tagOverrideHandlers?.get(tagName) ?? tagHandlers[tagId];
|
|
@@ -1311,6 +1410,7 @@ function processOpeningTag(tagName, tagId, htmlChunk, i, state, handleEvent) {
|
|
|
1311
1410
|
state.inDoubleQuote = false;
|
|
1312
1411
|
state.inBacktick = false;
|
|
1313
1412
|
state.lastCharWasBackslash = false;
|
|
1413
|
+
state.inRawTextQuoteAware = tag.tagId === 52 || tag.tagId === 53;
|
|
1314
1414
|
}
|
|
1315
1415
|
if (result.selfClosing) {
|
|
1316
1416
|
closeNode(tag, state, handleEvent);
|
|
@@ -1454,4 +1554,4 @@ function parseAttributes(attrStr) {
|
|
|
1454
1554
|
}
|
|
1455
1555
|
return result;
|
|
1456
1556
|
}
|
|
1457
|
-
export { buildTagOverrideHandlers
|
|
1557
|
+
export { buildTagOverrideHandlers, parseAttributes, parseHtml, parseHtmlStream };
|
package/dist/_chunks/plugins.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { TagIdMap } from "./const.mjs";
|
|
2
2
|
function createPlugin(plugin) {
|
|
3
3
|
return plugin;
|
|
4
4
|
}
|
|
@@ -216,14 +216,23 @@ function compileSelector(selector) {
|
|
|
216
216
|
};
|
|
217
217
|
return parseSelector(selector);
|
|
218
218
|
}
|
|
219
|
+
function isHidden(element) {
|
|
220
|
+
const style = element.attributes?.style;
|
|
221
|
+
if (style && (style.includes("display:none") || style.includes("display: none") || style.includes("visibility:hidden") || style.includes("visibility: hidden") || style.includes("position:absolute") || style.includes("position: absolute") || style.includes("position:fixed") || style.includes("position: fixed"))) return true;
|
|
222
|
+
const hidden = element.attributes?.hidden;
|
|
223
|
+
return hidden !== void 0 && hidden.toLowerCase() !== "until-found";
|
|
224
|
+
}
|
|
219
225
|
function filterPlugin(options = {}) {
|
|
220
226
|
const includeSelectors = options.include?.map((selector) => compileSelector(selector)) || [];
|
|
221
227
|
const excludeSelectors = options.exclude?.map((selector) => compileSelector(selector)) || [];
|
|
222
228
|
const processChildren = options.processChildren !== false;
|
|
229
|
+
const hiddenNodes = /* @__PURE__ */ new WeakSet();
|
|
223
230
|
return createPlugin({ beforeNodeProcess(event) {
|
|
224
231
|
const { node } = event;
|
|
225
232
|
if (node.type === 2) {
|
|
226
|
-
|
|
233
|
+
const parent = node.parent;
|
|
234
|
+
if (parent && hiddenNodes.has(parent)) return { skip: true };
|
|
235
|
+
let currentParent = parent;
|
|
227
236
|
while (currentParent && excludeSelectors.length) {
|
|
228
237
|
if (excludeSelectors.some((selector) => selector.matches(currentParent))) return { skip: true };
|
|
229
238
|
currentParent = currentParent.parent;
|
|
@@ -232,15 +241,14 @@ function filterPlugin(options = {}) {
|
|
|
232
241
|
}
|
|
233
242
|
if (node.type !== 1) return;
|
|
234
243
|
const element = node;
|
|
235
|
-
if (
|
|
236
|
-
|
|
237
|
-
|
|
244
|
+
if ((element.parent ? hiddenNodes.has(element.parent) : false) || isHidden(element)) {
|
|
245
|
+
hiddenNodes.add(element);
|
|
246
|
+
return { skip: true };
|
|
238
247
|
}
|
|
248
|
+
if (excludeSelectors.length && excludeSelectors.some((selector) => selector.matches(element))) return { skip: true };
|
|
239
249
|
let currentParent = element.parent;
|
|
240
|
-
while (currentParent) {
|
|
241
|
-
if (excludeSelectors.
|
|
242
|
-
if (excludeSelectors.some((selector) => selector.matches(currentParent))) return { skip: true };
|
|
243
|
-
}
|
|
250
|
+
while (currentParent && excludeSelectors.length) {
|
|
251
|
+
if (excludeSelectors.some((selector) => selector.matches(currentParent))) return { skip: true };
|
|
244
252
|
currentParent = currentParent.parent;
|
|
245
253
|
}
|
|
246
254
|
if (includeSelectors.length) {
|
|
@@ -660,4 +668,4 @@ function tailwindPlugin() {
|
|
|
660
668
|
}
|
|
661
669
|
});
|
|
662
670
|
}
|
|
663
|
-
export {
|
|
671
|
+
export { createPlugin, extractionCollectorPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { DEFAULT_BLOCK_SPACING, NO_SPACING } from "./const.mjs";
|
|
2
|
+
import { parseHtmlStream } from "./parse.mjs";
|
|
3
|
+
import { extractionCollectorPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin } from "./plugins.mjs";
|
|
4
4
|
function processPluginsForEvent(event, plugins, state, processEvent) {
|
|
5
5
|
if (plugins?.length) {
|
|
6
6
|
if (event.node.type === 1 && event.type === 0) {
|
|
@@ -85,6 +85,73 @@ function shouldAddSpacingBeforeText(lastChar, lastNode, textNode) {
|
|
|
85
85
|
if (firstChar === "." || firstChar === "," || firstChar === "!" || firstChar === "?" || firstChar === ":" || firstChar === ";" || firstChar === "_" || firstChar === "*" || firstChar === "`" || firstChar === ")" || firstChar === "]") return false;
|
|
86
86
|
return true;
|
|
87
87
|
}
|
|
88
|
+
function canWrapHere(depthMap) {
|
|
89
|
+
if (depthMap[34] || depthMap[23] || depthMap[32] || depthMap[31]) return false;
|
|
90
|
+
for (let h = 7; h <= 12; h++) if (depthMap[h]) return false;
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
function currentColumn(buffer) {
|
|
94
|
+
let col = 0;
|
|
95
|
+
for (let i = buffer.length - 1; i >= 0; i--) {
|
|
96
|
+
const s = buffer[i];
|
|
97
|
+
const nl = s.lastIndexOf("\n");
|
|
98
|
+
if (nl >= 0) return col + [...s.slice(nl + 1)].length;
|
|
99
|
+
col += [...s].length;
|
|
100
|
+
}
|
|
101
|
+
return col;
|
|
102
|
+
}
|
|
103
|
+
function wrapContinuationPrefix(state, node) {
|
|
104
|
+
const chain = [];
|
|
105
|
+
let cur = node.parent;
|
|
106
|
+
while (cur) {
|
|
107
|
+
chain.push(cur);
|
|
108
|
+
cur = cur.parent;
|
|
109
|
+
}
|
|
110
|
+
let p = "";
|
|
111
|
+
let liIdx = 0;
|
|
112
|
+
for (let i = chain.length - 1; i >= 0; i--) {
|
|
113
|
+
const tagId = chain[i].tagId;
|
|
114
|
+
if (tagId === 22) p += "> ";
|
|
115
|
+
else if (tagId === 25) {
|
|
116
|
+
const w = state.listIndentWidths[liIdx] ?? 2;
|
|
117
|
+
p += " ".repeat(w);
|
|
118
|
+
liIdx++;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return p;
|
|
122
|
+
}
|
|
123
|
+
function wrapText(value, col, width, prefix) {
|
|
124
|
+
const leading = value.charCodeAt(0) === 32;
|
|
125
|
+
const trailing = value.charCodeAt(value.length - 1) === 32;
|
|
126
|
+
const prefixLen = [...prefix].length;
|
|
127
|
+
let out = "";
|
|
128
|
+
let first = true;
|
|
129
|
+
let i = 0;
|
|
130
|
+
const len = value.length;
|
|
131
|
+
while (i < len) {
|
|
132
|
+
let next = value.indexOf(" ", i);
|
|
133
|
+
if (next === -1) next = len;
|
|
134
|
+
if (next > i) {
|
|
135
|
+
const word = value.slice(i, next);
|
|
136
|
+
const wordLen = [...word].length;
|
|
137
|
+
const needSpace = first ? leading : true;
|
|
138
|
+
if (needSpace && col > prefixLen && col + 1 + wordLen > width) {
|
|
139
|
+
out += `\n${prefix}`;
|
|
140
|
+
col = prefixLen;
|
|
141
|
+
} else if (needSpace) {
|
|
142
|
+
out += " ";
|
|
143
|
+
col += 1;
|
|
144
|
+
}
|
|
145
|
+
out += word;
|
|
146
|
+
col += wordLen;
|
|
147
|
+
first = false;
|
|
148
|
+
}
|
|
149
|
+
i = next + 1;
|
|
150
|
+
}
|
|
151
|
+
if (trailing && out !== "" && !out.endsWith(" ") && !out.endsWith("\n")) out += " ";
|
|
152
|
+
if (out === "" && (leading || trailing)) out = " ";
|
|
153
|
+
return out;
|
|
154
|
+
}
|
|
88
155
|
function calculateNewLineConfig(node) {
|
|
89
156
|
const tagId = node.tagId;
|
|
90
157
|
const depthMap = node.depthMap;
|
|
@@ -105,11 +172,26 @@ function calculateNewLineConfig(node) {
|
|
|
105
172
|
if (tagId === -1) return NO_SPACING;
|
|
106
173
|
return DEFAULT_BLOCK_SPACING;
|
|
107
174
|
}
|
|
175
|
+
function hasNonWhitespace(value) {
|
|
176
|
+
for (let i = 0; i < value.length; i++) {
|
|
177
|
+
const c = value.charCodeAt(i);
|
|
178
|
+
if (c !== 32 && c !== 9 && c !== 10 && c !== 13) return true;
|
|
179
|
+
}
|
|
180
|
+
return false;
|
|
181
|
+
}
|
|
182
|
+
function flushPreFence(state) {
|
|
183
|
+
state.preFencePending = false;
|
|
184
|
+
state.preOwnFence = true;
|
|
185
|
+
const lang = state.preFenceLang || "";
|
|
186
|
+
const fence = (state.depthMap[25] || 0) > 0 ? `\n\n${state.listIndent}\`\`\`${lang}\n${state.listIndent}` : `\`\`\`${lang}\n`;
|
|
187
|
+
state.buffer.push(fence);
|
|
188
|
+
state.lastContentCache = fence;
|
|
189
|
+
}
|
|
108
190
|
function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverrideHandlers) {
|
|
109
191
|
const state = {
|
|
110
192
|
options,
|
|
111
193
|
buffer: [],
|
|
112
|
-
depthMap: new Uint8Array(
|
|
194
|
+
depthMap: new Uint8Array(109),
|
|
113
195
|
listIndent: "",
|
|
114
196
|
listIndentWidths: []
|
|
115
197
|
};
|
|
@@ -120,6 +202,13 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
|
|
|
120
202
|
state.lastNode = event.node;
|
|
121
203
|
state.depth = node.depth;
|
|
122
204
|
const buff = state.buffer;
|
|
205
|
+
if (state.preFencePending && eventType === 0) {
|
|
206
|
+
if (node.type === 1) {
|
|
207
|
+
const el = node;
|
|
208
|
+
if (el.tagId === 23 && el.parent?.tagId === 34) state.preFencePending = false;
|
|
209
|
+
else if (el.tagId !== 34) flushPreFence(state);
|
|
210
|
+
} else if (node.type === 2 && hasNonWhitespace(node.value)) flushPreFence(state);
|
|
211
|
+
}
|
|
123
212
|
const lastBuffEntry = buff.at(-1);
|
|
124
213
|
const lastChar = lastBuffEntry?.charAt(lastBuffEntry.length - 1) || "";
|
|
125
214
|
let secondLastChar;
|
|
@@ -132,6 +221,7 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
|
|
|
132
221
|
const textNode = node;
|
|
133
222
|
if (textNode.value) {
|
|
134
223
|
if (textNode.excludedFromMarkdown) return;
|
|
224
|
+
if (state.preFencePending) return;
|
|
135
225
|
if (textNode.value === " " && lastChar === "\n") return;
|
|
136
226
|
if (shouldAddSpacingBeforeText(lastChar, lastNode, textNode)) textNode.value = ` ${textNode.value}`;
|
|
137
227
|
if ((state.depthMap[34] || 0) > 0 && (state.depthMap[25] || 0) > 0) {
|
|
@@ -140,8 +230,15 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
|
|
|
140
230
|
if (lastChar === "\n" && value[0] && value[0] !== "\n") value = indent + value;
|
|
141
231
|
textNode.value = value;
|
|
142
232
|
}
|
|
143
|
-
state.
|
|
144
|
-
state.
|
|
233
|
+
const wrapWidth = state.options?.wrapWidth;
|
|
234
|
+
if (wrapWidth && canWrapHere(state.depthMap)) {
|
|
235
|
+
const wrapped = wrapText(textNode.value, currentColumn(state.buffer), wrapWidth, wrapContinuationPrefix(state, textNode));
|
|
236
|
+
state.buffer.push(wrapped);
|
|
237
|
+
state.lastContentCache = wrapped;
|
|
238
|
+
} else {
|
|
239
|
+
state.buffer.push(textNode.value);
|
|
240
|
+
state.lastContentCache = textNode.value;
|
|
241
|
+
}
|
|
145
242
|
}
|
|
146
243
|
state.lastTextNode = textNode;
|
|
147
244
|
return;
|
|
@@ -282,4 +379,4 @@ function resolvePlugins(options, hooks) {
|
|
|
282
379
|
frontmatterCallback
|
|
283
380
|
};
|
|
284
381
|
}
|
|
285
|
-
export { createMarkdownProcessor
|
|
382
|
+
export { createMarkdownProcessor, processPluginsForEvent, resolvePlugins };
|
package/dist/_chunks/src.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import "./const.mjs";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { buildTagOverrideHandlers, parseHtmlStream } from "./parse.mjs";
|
|
3
|
+
import { createMarkdownProcessor, processPluginsForEvent, resolvePlugins } from "./resolve-plugins.mjs";
|
|
4
4
|
import "./plugins.mjs";
|
|
5
5
|
import "../preset/minimal.mjs";
|
|
6
6
|
function resolveClean(clean) {
|
|
@@ -343,4 +343,4 @@ function streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
|
343
343
|
const { plugins } = resolvePlugins(options, resolveHooks(options));
|
|
344
344
|
return streamHtmlToMarkdown$1(htmlStream, options, plugins, options.plugins?.tagOverrides ? buildTagOverrideHandlers(options.plugins.tagOverrides) : void 0);
|
|
345
345
|
}
|
|
346
|
-
export {
|
|
346
|
+
export { htmlToMarkdown, streamHtmlToMarkdown };
|
package/dist/_chunks/types.d.mts
CHANGED
|
@@ -137,6 +137,13 @@ interface EngineOptions {
|
|
|
137
137
|
* final markdown (sync API only for `fragments`).
|
|
138
138
|
*/
|
|
139
139
|
clean?: boolean | CleanOptions;
|
|
140
|
+
/**
|
|
141
|
+
* Hard-wrap prose at this many characters, breaking on word boundaries.
|
|
142
|
+
* Applied inline during conversion (zero-cost when unset). Code blocks
|
|
143
|
+
* (`<pre>`/`<code>`), tables, and headings are never wrapped. `0` disables
|
|
144
|
+
* wrapping.
|
|
145
|
+
*/
|
|
146
|
+
wrapWidth?: number;
|
|
140
147
|
}
|
|
141
148
|
interface ElementNode extends Node {
|
|
142
149
|
/** Element tag name (for ELEMENT_NODE) */
|
|
@@ -251,6 +258,12 @@ interface MdreamRuntimeState extends Partial<MdreamProcessingState> {
|
|
|
251
258
|
*/
|
|
252
259
|
listIndent?: string;
|
|
253
260
|
listIndentWidths?: number[];
|
|
261
|
+
/**
|
|
262
|
+
* <pre> fenced-code deferral (issue #97). See MarkdownState for semantics.
|
|
263
|
+
*/
|
|
264
|
+
preFencePending?: boolean;
|
|
265
|
+
preFenceLang?: string;
|
|
266
|
+
preOwnFence?: boolean;
|
|
254
267
|
}
|
|
255
268
|
type NodeEventEnter = 0;
|
|
256
269
|
type NodeEventExit = 1;
|
|
@@ -396,4 +409,4 @@ interface SplitterOptions extends EngineOptions {
|
|
|
396
409
|
*/
|
|
397
410
|
keepSeparator?: boolean;
|
|
398
411
|
}
|
|
399
|
-
export {
|
|
412
|
+
export { BuiltinPlugins, CleanOptions, ELEMENT_NODE, ElementNode, EngineOptions, ExtractedElement, FrontmatterConfig, MarkdownChunk, MdreamOptions, MdreamRuntimeState, Node, NodeEvent, NodeEventEnter$1 as NodeEventEnter, NodeEventExit$1 as NodeEventExit, PluginContext, SplitterOptions, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, TagHandler, TagOverride, TextNode, TransformPlugin };
|
package/dist/cli.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { streamHtmlToMarkdown } from "./_chunks/src.mjs";
|
|
2
2
|
import { withMinimalPreset } from "./preset/minimal.mjs";
|
|
3
3
|
import { readFileSync } from "node:fs";
|
|
4
4
|
import { Readable } from "node:stream";
|
|
@@ -6,7 +6,10 @@ import { fileURLToPath } from "node:url";
|
|
|
6
6
|
import { cac } from "cac";
|
|
7
7
|
import { dirname, join } from "pathe";
|
|
8
8
|
async function streamingConvert(options = {}) {
|
|
9
|
-
let conversionOptions = {
|
|
9
|
+
let conversionOptions = {
|
|
10
|
+
origin: options.origin,
|
|
11
|
+
wrapWidth: options.wrapWidth ? Number(options.wrapWidth) || void 0 : void 0
|
|
12
|
+
};
|
|
10
13
|
if (options.preset === "minimal") conversionOptions = withMinimalPreset(conversionOptions);
|
|
11
14
|
const markdownGenerator = streamHtmlToMarkdown(Readable.toWeb(process.stdin), conversionOptions);
|
|
12
15
|
for await (const markdownChunk of markdownGenerator) if (markdownChunk && markdownChunk.length > 0) process.stdout.write(markdownChunk);
|
|
@@ -14,7 +17,7 @@ async function streamingConvert(options = {}) {
|
|
|
14
17
|
const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
|
|
15
18
|
const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
|
|
16
19
|
const cli = cac();
|
|
17
|
-
cli.command("[options]", "Convert HTML from stdin to Markdown on stdout (JS engine)").option("--origin <url>", "Origin URL for resolving relative image paths").option("--preset <preset>", "Conversion presets: minimal").action(async (_, opts) => {
|
|
20
|
+
cli.command("[options]", "Convert HTML from stdin to Markdown on stdout (JS engine)").option("--origin <url>", "Origin URL for resolving relative image paths").option("--preset <preset>", "Conversion presets: minimal").option("--wrap-width <n>", "Hard-wrap prose at <n> characters on word boundaries").action(async (_, opts) => {
|
|
18
21
|
await streamingConvert(opts);
|
|
19
22
|
});
|
|
20
23
|
cli.help().version(packageJson.version).parse();
|
package/dist/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { BuiltinPlugins, CleanOptions, ELEMENT_NODE, ElementNode, EngineOptions, ExtractedElement, FrontmatterConfig, MarkdownChunk, MdreamOptions, Node, NodeEvent, NodeEventEnter, NodeEventExit, PluginContext, SplitterOptions, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, TagOverride, TextNode, TransformPlugin } from "./_chunks/types.mjs";
|
|
2
|
+
import { createPlugin, htmlToMarkdown, streamHtmlToMarkdown } from "./_chunks/index.mjs";
|
|
3
|
+
import { withMinimalPreset } from "./_chunks/minimal.mjs";
|
|
4
4
|
export { type BuiltinPlugins, type CleanOptions, ELEMENT_NODE, type ElementNode, type EngineOptions, type ExtractedElement, type FrontmatterConfig, type MarkdownChunk, type MdreamOptions, type Node, type NodeEvent, NodeEventEnter, NodeEventExit, type PluginContext, type SplitterOptions, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, type TagOverride, type TextNode, type TransformPlugin, createPlugin, htmlToMarkdown, streamHtmlToMarkdown, withMinimalPreset };
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { htmlToMarkdown, streamHtmlToMarkdown } from "./_chunks/src.mjs";
|
|
2
|
+
import { ELEMENT_NODE, NodeEventEnter, NodeEventExit, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE } from "./_chunks/const.mjs";
|
|
3
|
+
import { createPlugin } from "./_chunks/plugins.mjs";
|
|
4
4
|
import { withMinimalPreset } from "./preset/minimal.mjs";
|
|
5
5
|
export { ELEMENT_NODE, NodeEventEnter, NodeEventExit, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, createPlugin, htmlToMarkdown, streamHtmlToMarkdown, withMinimalPreset };
|
package/dist/parse.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ElementNode, Node, NodeEvent, TagHandler, TransformPlugin } from "./_chunks/types.mjs";
|
|
2
2
|
interface ParseOptions {
|
|
3
3
|
resolvedPlugins?: TransformPlugin[];
|
|
4
4
|
}
|
|
@@ -27,6 +27,13 @@ interface ParseState {
|
|
|
27
27
|
inSingleQuote?: boolean;
|
|
28
28
|
inDoubleQuote?: boolean;
|
|
29
29
|
inBacktick?: boolean;
|
|
30
|
+
/**
|
|
31
|
+
* Whether the current non-nesting tag is quote-aware rawtext (script/style only).
|
|
32
|
+
* For these, a `</script>`/`</style>` inside a JS/CSS string literal must not close
|
|
33
|
+
* the tag. Other non-nesting tags (template, textarea, ...) hold literal HTML/text
|
|
34
|
+
* where apostrophes are not string delimiters, so their closing tag always wins.
|
|
35
|
+
*/
|
|
36
|
+
inRawTextQuoteAware?: boolean;
|
|
30
37
|
/** Backslash escaping state tracking - avoids checking previous character */
|
|
31
38
|
lastCharWasBackslash?: boolean;
|
|
32
39
|
/** Resolved plugin instances for event processing */
|
package/dist/parse.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { parseAttributes, parseHtml, parseHtmlStream } from "./_chunks/parse.mjs";
|
|
2
2
|
export { parseAttributes, parseHtml, parseHtmlStream };
|
package/dist/plugins.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { ElementNode, ExtractedElement as ExtractedElement$1, MdreamRuntimeState, TransformPlugin } from "./_chunks/types.mjs";
|
|
2
|
+
import { createPlugin } from "./_chunks/index.mjs";
|
|
3
3
|
interface ExtractedElement extends ElementNode {
|
|
4
4
|
textContent: string;
|
|
5
5
|
}
|
package/dist/plugins.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createPlugin, extractionCollectorPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin } from "./_chunks/plugins.mjs";
|
|
2
2
|
export { createPlugin, extractionCollectorPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin };
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { withMinimalPreset } from "../_chunks/minimal.mjs";
|
|
2
2
|
export { withMinimalPreset };
|
package/dist/splitter.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MarkdownChunk, SplitterOptions } from "./_chunks/types.mjs";
|
|
2
2
|
/**
|
|
3
3
|
* Convert HTML to Markdown and split into chunks in single pass.
|
|
4
4
|
* Yields chunks during HTML event processing for better memory efficiency.
|
package/dist/splitter.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import "./_chunks/const.mjs";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { parseHtmlStream } from "./_chunks/parse.mjs";
|
|
3
|
+
import { createMarkdownProcessor, processPluginsForEvent, resolvePlugins } from "./_chunks/resolve-plugins.mjs";
|
|
4
4
|
const MARKDOWN_HEADER_LINE_RE = /^#{1,6}\s+/;
|
|
5
5
|
const NEWLINE_RE = /\n/g;
|
|
6
6
|
const DEFAULT_HEADERS_TO_SPLIT_ON = [
|
package/package.json
CHANGED