@mdream/js 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/const.mjs +3 -2
- package/dist/_chunks/index.d.mts +2 -2
- package/dist/_chunks/minimal.d.mts +2 -2
- package/dist/_chunks/parse.mjs +113 -12
- package/dist/_chunks/plugins.mjs +18 -10
- package/dist/_chunks/resolve-plugins.mjs +29 -6
- package/dist/_chunks/src.mjs +3 -3
- package/dist/_chunks/types.d.mts +13 -1
- package/dist/cli.mjs +1 -1
- package/dist/index.d.mts +3 -3
- package/dist/index.mjs +3 -3
- package/dist/parse.d.mts +8 -1
- package/dist/parse.mjs +1 -1
- package/dist/plugins.d.mts +2 -2
- package/dist/plugins.mjs +1 -1
- package/dist/preset/minimal.d.mts +1 -1
- package/dist/splitter.d.mts +1 -1
- package/dist/splitter.mjs +2 -2
- package/package.json +1 -1
package/dist/_chunks/const.mjs
CHANGED
|
@@ -116,7 +116,8 @@ const TagIdMap = {
|
|
|
116
116
|
main: 104,
|
|
117
117
|
header: 105,
|
|
118
118
|
figcaption: 106,
|
|
119
|
-
caption: 107
|
|
119
|
+
caption: 107,
|
|
120
|
+
datalist: 108
|
|
120
121
|
};
|
|
121
122
|
Object.entries(TagIdMap).map(([name, id]) => [id, name]);
|
|
122
123
|
const NO_SPACING = [0, 0];
|
|
@@ -124,4 +125,4 @@ const DEFAULT_BLOCK_SPACING = [2, 2];
|
|
|
124
125
|
const BLOCKQUOTE_SPACING = [1, 1];
|
|
125
126
|
const LIST_ITEM_SPACING = [1, 0];
|
|
126
127
|
const TABLE_ROW_SPACING = [0, 1];
|
|
127
|
-
export {
|
|
128
|
+
export { BLOCKQUOTE_SPACING, DEFAULT_BLOCK_SPACING, ELEMENT_NODE, LIST_ITEM_SPACING, NO_SPACING, NodeEventEnter, NodeEventExit, TABLE_ROW_SPACING, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, TagIdMap };
|
package/dist/_chunks/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MdreamOptions, TransformPlugin } from "./types.mjs";
|
|
2
2
|
/**
|
|
3
3
|
* Create a plugin with type-safe hook definitions.
|
|
4
4
|
* All TransformPlugin fields are optional, so this is a typed identity function.
|
|
@@ -6,4 +6,4 @@ import { _ as TransformPlugin, c as MdreamOptions } from "./types.mjs";
|
|
|
6
6
|
declare function createPlugin(plugin: TransformPlugin): TransformPlugin;
|
|
7
7
|
declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
|
|
8
8
|
declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
9
|
-
export {
|
|
9
|
+
export { createPlugin, htmlToMarkdown, streamHtmlToMarkdown };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { EngineOptions } from "./types.mjs";
|
|
2
2
|
/**
|
|
3
3
|
* Creates a configurable minimal preset with advanced options.
|
|
4
4
|
* Returns declarative plugin config that works with both JS and Rust engines.
|
|
5
5
|
*/
|
|
6
6
|
declare function withMinimalPreset<T extends EngineOptions>(options?: T): T;
|
|
7
|
-
export { withMinimalPreset
|
|
7
|
+
export { withMinimalPreset };
|
package/dist/_chunks/parse.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { BLOCKQUOTE_SPACING, LIST_ITEM_SPACING, NO_SPACING, TABLE_ROW_SPACING, TagIdMap } from "./const.mjs";
|
|
2
2
|
function resolveUrl(url, origin) {
|
|
3
3
|
if (!url) return url;
|
|
4
4
|
if (url.startsWith("//")) return `https:${url}`;
|
|
@@ -155,9 +155,29 @@ const tagHandlers = {
|
|
|
155
155
|
},
|
|
156
156
|
spacing: BLOCKQUOTE_SPACING
|
|
157
157
|
},
|
|
158
|
+
[34]: {
|
|
159
|
+
enter: ({ node, state }) => {
|
|
160
|
+
state.preFencePending = true;
|
|
161
|
+
state.preOwnFence = false;
|
|
162
|
+
state.preFenceLang = getLanguageFromClass(node.attributes?.class);
|
|
163
|
+
},
|
|
164
|
+
exit: ({ node, state }) => {
|
|
165
|
+
const ownFence = state.preOwnFence;
|
|
166
|
+
state.preFencePending = false;
|
|
167
|
+
state.preOwnFence = false;
|
|
168
|
+
if (!ownFence) return;
|
|
169
|
+
if ((node.depthMap[25] || 0) > 0) {
|
|
170
|
+
const indent = state.listIndent;
|
|
171
|
+
return `\n${indent}\`\`\`\n\n${indent}`;
|
|
172
|
+
}
|
|
173
|
+
return `\n\`\`\``;
|
|
174
|
+
}
|
|
175
|
+
},
|
|
158
176
|
[23]: {
|
|
159
177
|
enter: ({ node, state }) => {
|
|
160
178
|
if ((node.depthMap[34] || 0) > 0) {
|
|
179
|
+
if (state.preOwnFence) return;
|
|
180
|
+
state.preFencePending = false;
|
|
161
181
|
const language = getLanguageFromClass(node.attributes?.class);
|
|
162
182
|
if ((node.depthMap[25] || 0) > 0) return `\n\n${state.listIndent}\`\`\`${language}\n`;
|
|
163
183
|
return `\`\`\`${language}\n`;
|
|
@@ -171,6 +191,7 @@ const tagHandlers = {
|
|
|
171
191
|
},
|
|
172
192
|
exit: ({ node, state }) => {
|
|
173
193
|
if ((node.depthMap[34] || 0) > 0) {
|
|
194
|
+
if (state.preOwnFence) return;
|
|
174
195
|
if ((node.depthMap[25] || 0) > 0) {
|
|
175
196
|
const indent = state.listIndent;
|
|
176
197
|
return `\n${indent}\`\`\`\n\n${indent}`;
|
|
@@ -449,7 +470,11 @@ const tagHandlers = {
|
|
|
449
470
|
[75]: { spacing: NO_SPACING },
|
|
450
471
|
[76]: { spacing: NO_SPACING },
|
|
451
472
|
[77]: { spacing: NO_SPACING },
|
|
452
|
-
[78]: {
|
|
473
|
+
[78]: {
|
|
474
|
+
isNonNesting: true,
|
|
475
|
+
excludesTextNodes: true,
|
|
476
|
+
spacing: NO_SPACING
|
|
477
|
+
},
|
|
453
478
|
[79]: {
|
|
454
479
|
enter: () => "",
|
|
455
480
|
exit: () => "",
|
|
@@ -489,6 +514,11 @@ const tagHandlers = {
|
|
|
489
514
|
excludesTextNodes: true,
|
|
490
515
|
spacing: NO_SPACING
|
|
491
516
|
},
|
|
517
|
+
[108]: {
|
|
518
|
+
isNonNesting: true,
|
|
519
|
+
excludesTextNodes: true,
|
|
520
|
+
spacing: NO_SPACING
|
|
521
|
+
},
|
|
492
522
|
[85]: {
|
|
493
523
|
isNonNesting: true,
|
|
494
524
|
spacing: NO_SPACING
|
|
@@ -622,6 +652,7 @@ function buildTagOverrideHandlers(overrides) {
|
|
|
622
652
|
if (override.enter !== void 0) {
|
|
623
653
|
const enterStr = override.enter;
|
|
624
654
|
handler.enter = () => enterStr;
|
|
655
|
+
handler.literalEnter = true;
|
|
625
656
|
}
|
|
626
657
|
if (override.exit !== void 0) {
|
|
627
658
|
const exitStr = override.exit;
|
|
@@ -963,7 +994,7 @@ function parseHtml(html, options = {}) {
|
|
|
963
994
|
return {
|
|
964
995
|
events,
|
|
965
996
|
remainingHtml: parseHtmlInternal(html, {
|
|
966
|
-
depthMap: new Uint8Array(
|
|
997
|
+
depthMap: new Uint8Array(109),
|
|
967
998
|
depth: 0,
|
|
968
999
|
resolvedPlugins: options.resolvedPlugins || []
|
|
969
1000
|
}, (event) => {
|
|
@@ -976,7 +1007,7 @@ function parseHtmlStream(html, state, onEvent) {
|
|
|
976
1007
|
}
|
|
977
1008
|
function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
978
1009
|
let textBuffer = "";
|
|
979
|
-
state.depthMap ??= new Uint8Array(
|
|
1010
|
+
state.depthMap ??= new Uint8Array(109);
|
|
980
1011
|
state.depth ??= 0;
|
|
981
1012
|
state.lastCharWasWhitespace ??= true;
|
|
982
1013
|
state.justClosedTag ??= false;
|
|
@@ -1013,7 +1044,7 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1013
1044
|
else if (currentCharCode === CLOSE_BRACKET_CHAR && state.depthMap[26]) textBuffer += "\\]";
|
|
1014
1045
|
else if (currentCharCode === GT_CHAR && state.depthMap[22]) textBuffer += "\\>";
|
|
1015
1046
|
else textBuffer += htmlChunk[i];
|
|
1016
|
-
if (state.
|
|
1047
|
+
if (state.inRawTextQuoteAware) {
|
|
1017
1048
|
if (!state.lastCharWasBackslash) {
|
|
1018
1049
|
if (currentCharCode === APOS_CHAR && !state.inDoubleQuote && !state.inBacktick) state.inSingleQuote = !state.inSingleQuote;
|
|
1019
1050
|
else if (currentCharCode === QUOTE_CHAR && !state.inSingleQuote && !state.inBacktick) state.inDoubleQuote = !state.inDoubleQuote;
|
|
@@ -1031,6 +1062,26 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1031
1062
|
}
|
|
1032
1063
|
const nextCharCode = htmlChunk.charCodeAt(i + 1);
|
|
1033
1064
|
if (nextCharCode === EXCLAMATION_CHAR) {
|
|
1065
|
+
if (htmlChunk.charCodeAt(i + 2) === OPEN_BRACKET_CHAR) {
|
|
1066
|
+
if (htmlChunk.startsWith("<![CDATA[", i)) {
|
|
1067
|
+
const end = htmlChunk.indexOf("]]>", i + 9);
|
|
1068
|
+
if (end === -1) {
|
|
1069
|
+
textBuffer += htmlChunk.substring(i);
|
|
1070
|
+
break;
|
|
1071
|
+
}
|
|
1072
|
+
if (textBuffer.length > 0) {
|
|
1073
|
+
processTextBuffer(textBuffer, state, handleEvent);
|
|
1074
|
+
textBuffer = "";
|
|
1075
|
+
}
|
|
1076
|
+
processCdataSection(htmlChunk.substring(i + 9, end), state, handleEvent);
|
|
1077
|
+
i = end + 3;
|
|
1078
|
+
continue;
|
|
1079
|
+
}
|
|
1080
|
+
if (chunkLength - i < 9 && "<![CDATA[".startsWith(htmlChunk.substring(i))) {
|
|
1081
|
+
textBuffer += htmlChunk.substring(i);
|
|
1082
|
+
break;
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1034
1085
|
if (textBuffer.length > 0) {
|
|
1035
1086
|
processTextBuffer(textBuffer, state, handleEvent);
|
|
1036
1087
|
textBuffer = "";
|
|
@@ -1043,7 +1094,7 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1043
1094
|
}
|
|
1044
1095
|
} else if (nextCharCode === SLASH_CHAR) {
|
|
1045
1096
|
if (state.currentNode?.tagHandler?.isNonNesting) {
|
|
1046
|
-
if (state.inSingleQuote || state.inDoubleQuote || state.inBacktick) {
|
|
1097
|
+
if (state.inRawTextQuoteAware && (state.inSingleQuote || state.inDoubleQuote || state.inBacktick)) {
|
|
1047
1098
|
textBuffer += htmlChunk[i];
|
|
1048
1099
|
i++;
|
|
1049
1100
|
continue;
|
|
@@ -1089,10 +1140,8 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1089
1140
|
const tagId = TagIdMap[tagName] ?? -1;
|
|
1090
1141
|
i2 = tagNameEnd;
|
|
1091
1142
|
if (state.currentNode?.tagHandler?.isNonNesting) {
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
continue;
|
|
1095
|
-
}
|
|
1143
|
+
textBuffer += htmlChunk[i++];
|
|
1144
|
+
continue;
|
|
1096
1145
|
}
|
|
1097
1146
|
if (!tagName) {
|
|
1098
1147
|
textBuffer += htmlChunk[i++];
|
|
@@ -1113,6 +1162,10 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1113
1162
|
}
|
|
1114
1163
|
}
|
|
1115
1164
|
}
|
|
1165
|
+
state.inSingleQuote = false;
|
|
1166
|
+
state.inDoubleQuote = false;
|
|
1167
|
+
state.inBacktick = false;
|
|
1168
|
+
state.lastCharWasBackslash = false;
|
|
1116
1169
|
return textBuffer;
|
|
1117
1170
|
}
|
|
1118
1171
|
function processTextBuffer(textBuffer, state, handleEvent) {
|
|
@@ -1120,7 +1173,30 @@ function processTextBuffer(textBuffer, state, handleEvent) {
|
|
|
1120
1173
|
const containsWhitespace = state.textBufferContainsWhitespace;
|
|
1121
1174
|
state.textBufferContainsNonWhitespace = false;
|
|
1122
1175
|
state.textBufferContainsWhitespace = false;
|
|
1123
|
-
if (!state.currentNode)
|
|
1176
|
+
if (!state.currentNode) {
|
|
1177
|
+
if (!containsNonWhitespace) return;
|
|
1178
|
+
let rootText = textBuffer;
|
|
1179
|
+
if (rootText.length === 0) return;
|
|
1180
|
+
if (state.hasEncodedHtmlEntity) {
|
|
1181
|
+
rootText = decodeHTMLEntities(String(rootText));
|
|
1182
|
+
state.hasEncodedHtmlEntity = false;
|
|
1183
|
+
}
|
|
1184
|
+
const rootTextNode = {
|
|
1185
|
+
type: 2,
|
|
1186
|
+
value: rootText,
|
|
1187
|
+
parent: null,
|
|
1188
|
+
index: 0,
|
|
1189
|
+
depth: state.depth,
|
|
1190
|
+
containsWhitespace,
|
|
1191
|
+
excludedFromMarkdown: false
|
|
1192
|
+
};
|
|
1193
|
+
handleEvent({
|
|
1194
|
+
type: 0,
|
|
1195
|
+
node: rootTextNode
|
|
1196
|
+
});
|
|
1197
|
+
state.lastTextNode = rootTextNode;
|
|
1198
|
+
return;
|
|
1199
|
+
}
|
|
1124
1200
|
const excludesTextNodes = state.currentNode?.tagHandler?.excludesTextNodes;
|
|
1125
1201
|
const inPreTag = (state.depthMap[34] || 0) > 0;
|
|
1126
1202
|
if (!inPreTag && !containsNonWhitespace && !state.currentNode.childTextNodeIndex) return;
|
|
@@ -1218,6 +1294,7 @@ function closeNode(node, state, handleEvent) {
|
|
|
1218
1294
|
state.inDoubleQuote = false;
|
|
1219
1295
|
state.inBacktick = false;
|
|
1220
1296
|
state.lastCharWasBackslash = false;
|
|
1297
|
+
state.inRawTextQuoteAware = false;
|
|
1221
1298
|
}
|
|
1222
1299
|
state.depth--;
|
|
1223
1300
|
handleEvent({
|
|
@@ -1269,6 +1346,29 @@ function processCommentOrDoctype(htmlChunk, position) {
|
|
|
1269
1346
|
};
|
|
1270
1347
|
}
|
|
1271
1348
|
}
|
|
1349
|
+
function processCdataSection(content, state, handleEvent) {
|
|
1350
|
+
if (!state.tagOverrideHandlers?.has("#cdata-section")) return;
|
|
1351
|
+
const open = processOpeningTag("#cdata-section", -1, ">", 0, state, handleEvent);
|
|
1352
|
+
if (!open.complete || open.selfClosing) return;
|
|
1353
|
+
const node = state.currentNode;
|
|
1354
|
+
if (content.length > 0 && !node.tagHandler?.excludesTextNodes) {
|
|
1355
|
+
const textNode = {
|
|
1356
|
+
type: 2,
|
|
1357
|
+
value: content,
|
|
1358
|
+
parent: node,
|
|
1359
|
+
index: node.currentWalkIndex++,
|
|
1360
|
+
depth: state.depth,
|
|
1361
|
+
containsWhitespace: false
|
|
1362
|
+
};
|
|
1363
|
+
for (const parent of traverseUpToFirstBlockNode(node)) parent.childTextNodeIndex = (parent.childTextNodeIndex || 0) + 1;
|
|
1364
|
+
handleEvent({
|
|
1365
|
+
type: 0,
|
|
1366
|
+
node: textNode
|
|
1367
|
+
});
|
|
1368
|
+
state.lastTextNode = textNode;
|
|
1369
|
+
}
|
|
1370
|
+
closeNode(state.currentNode ?? null, state, handleEvent);
|
|
1371
|
+
}
|
|
1272
1372
|
function processOpeningTag(tagName, tagId, htmlChunk, i, state, handleEvent) {
|
|
1273
1373
|
if (state.currentNode?.tagHandler?.isNonNesting) closeNode(state.currentNode, state, handleEvent);
|
|
1274
1374
|
const tagHandler = state.tagOverrideHandlers?.get(tagName) ?? tagHandlers[tagId];
|
|
@@ -1310,6 +1410,7 @@ function processOpeningTag(tagName, tagId, htmlChunk, i, state, handleEvent) {
|
|
|
1310
1410
|
state.inDoubleQuote = false;
|
|
1311
1411
|
state.inBacktick = false;
|
|
1312
1412
|
state.lastCharWasBackslash = false;
|
|
1413
|
+
state.inRawTextQuoteAware = tag.tagId === 52 || tag.tagId === 53;
|
|
1313
1414
|
}
|
|
1314
1415
|
if (result.selfClosing) {
|
|
1315
1416
|
closeNode(tag, state, handleEvent);
|
|
@@ -1453,4 +1554,4 @@ function parseAttributes(attrStr) {
|
|
|
1453
1554
|
}
|
|
1454
1555
|
return result;
|
|
1455
1556
|
}
|
|
1456
|
-
export { buildTagOverrideHandlers
|
|
1557
|
+
export { buildTagOverrideHandlers, parseAttributes, parseHtml, parseHtmlStream };
|
package/dist/_chunks/plugins.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { TagIdMap } from "./const.mjs";
|
|
2
2
|
function createPlugin(plugin) {
|
|
3
3
|
return plugin;
|
|
4
4
|
}
|
|
@@ -216,14 +216,23 @@ function compileSelector(selector) {
|
|
|
216
216
|
};
|
|
217
217
|
return parseSelector(selector);
|
|
218
218
|
}
|
|
219
|
+
function isHidden(element) {
|
|
220
|
+
const style = element.attributes?.style;
|
|
221
|
+
if (style && (style.includes("display:none") || style.includes("display: none") || style.includes("visibility:hidden") || style.includes("visibility: hidden") || style.includes("position:absolute") || style.includes("position: absolute") || style.includes("position:fixed") || style.includes("position: fixed"))) return true;
|
|
222
|
+
const hidden = element.attributes?.hidden;
|
|
223
|
+
return hidden !== void 0 && hidden.toLowerCase() !== "until-found";
|
|
224
|
+
}
|
|
219
225
|
function filterPlugin(options = {}) {
|
|
220
226
|
const includeSelectors = options.include?.map((selector) => compileSelector(selector)) || [];
|
|
221
227
|
const excludeSelectors = options.exclude?.map((selector) => compileSelector(selector)) || [];
|
|
222
228
|
const processChildren = options.processChildren !== false;
|
|
229
|
+
const hiddenNodes = /* @__PURE__ */ new WeakSet();
|
|
223
230
|
return createPlugin({ beforeNodeProcess(event) {
|
|
224
231
|
const { node } = event;
|
|
225
232
|
if (node.type === 2) {
|
|
226
|
-
|
|
233
|
+
const parent = node.parent;
|
|
234
|
+
if (parent && hiddenNodes.has(parent)) return { skip: true };
|
|
235
|
+
let currentParent = parent;
|
|
227
236
|
while (currentParent && excludeSelectors.length) {
|
|
228
237
|
if (excludeSelectors.some((selector) => selector.matches(currentParent))) return { skip: true };
|
|
229
238
|
currentParent = currentParent.parent;
|
|
@@ -232,15 +241,14 @@ function filterPlugin(options = {}) {
|
|
|
232
241
|
}
|
|
233
242
|
if (node.type !== 1) return;
|
|
234
243
|
const element = node;
|
|
235
|
-
if (
|
|
236
|
-
|
|
237
|
-
|
|
244
|
+
if ((element.parent ? hiddenNodes.has(element.parent) : false) || isHidden(element)) {
|
|
245
|
+
hiddenNodes.add(element);
|
|
246
|
+
return { skip: true };
|
|
238
247
|
}
|
|
248
|
+
if (excludeSelectors.length && excludeSelectors.some((selector) => selector.matches(element))) return { skip: true };
|
|
239
249
|
let currentParent = element.parent;
|
|
240
|
-
while (currentParent) {
|
|
241
|
-
if (excludeSelectors.
|
|
242
|
-
if (excludeSelectors.some((selector) => selector.matches(currentParent))) return { skip: true };
|
|
243
|
-
}
|
|
250
|
+
while (currentParent && excludeSelectors.length) {
|
|
251
|
+
if (excludeSelectors.some((selector) => selector.matches(currentParent))) return { skip: true };
|
|
244
252
|
currentParent = currentParent.parent;
|
|
245
253
|
}
|
|
246
254
|
if (includeSelectors.length) {
|
|
@@ -660,4 +668,4 @@ function tailwindPlugin() {
|
|
|
660
668
|
}
|
|
661
669
|
});
|
|
662
670
|
}
|
|
663
|
-
export {
|
|
671
|
+
export { createPlugin, extractionCollectorPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { DEFAULT_BLOCK_SPACING, NO_SPACING } from "./const.mjs";
|
|
2
|
+
import { parseHtmlStream } from "./parse.mjs";
|
|
3
|
+
import { extractionCollectorPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin } from "./plugins.mjs";
|
|
4
4
|
function processPluginsForEvent(event, plugins, state, processEvent) {
|
|
5
5
|
if (plugins?.length) {
|
|
6
6
|
if (event.node.type === 1 && event.type === 0) {
|
|
@@ -105,11 +105,26 @@ function calculateNewLineConfig(node) {
|
|
|
105
105
|
if (tagId === -1) return NO_SPACING;
|
|
106
106
|
return DEFAULT_BLOCK_SPACING;
|
|
107
107
|
}
|
|
108
|
+
function hasNonWhitespace(value) {
|
|
109
|
+
for (let i = 0; i < value.length; i++) {
|
|
110
|
+
const c = value.charCodeAt(i);
|
|
111
|
+
if (c !== 32 && c !== 9 && c !== 10 && c !== 13) return true;
|
|
112
|
+
}
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
function flushPreFence(state) {
|
|
116
|
+
state.preFencePending = false;
|
|
117
|
+
state.preOwnFence = true;
|
|
118
|
+
const lang = state.preFenceLang || "";
|
|
119
|
+
const fence = (state.depthMap[25] || 0) > 0 ? `\n\n${state.listIndent}\`\`\`${lang}\n${state.listIndent}` : `\`\`\`${lang}\n`;
|
|
120
|
+
state.buffer.push(fence);
|
|
121
|
+
state.lastContentCache = fence;
|
|
122
|
+
}
|
|
108
123
|
function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverrideHandlers) {
|
|
109
124
|
const state = {
|
|
110
125
|
options,
|
|
111
126
|
buffer: [],
|
|
112
|
-
depthMap: new Uint8Array(
|
|
127
|
+
depthMap: new Uint8Array(109),
|
|
113
128
|
listIndent: "",
|
|
114
129
|
listIndentWidths: []
|
|
115
130
|
};
|
|
@@ -120,6 +135,13 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
|
|
|
120
135
|
state.lastNode = event.node;
|
|
121
136
|
state.depth = node.depth;
|
|
122
137
|
const buff = state.buffer;
|
|
138
|
+
if (state.preFencePending && eventType === 0) {
|
|
139
|
+
if (node.type === 1) {
|
|
140
|
+
const el = node;
|
|
141
|
+
if (el.tagId === 23 && el.parent?.tagId === 34) state.preFencePending = false;
|
|
142
|
+
else if (el.tagId !== 34) flushPreFence(state);
|
|
143
|
+
} else if (node.type === 2 && hasNonWhitespace(node.value)) flushPreFence(state);
|
|
144
|
+
}
|
|
123
145
|
const lastBuffEntry = buff.at(-1);
|
|
124
146
|
const lastChar = lastBuffEntry?.charAt(lastBuffEntry.length - 1) || "";
|
|
125
147
|
let secondLastChar;
|
|
@@ -132,6 +154,7 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
|
|
|
132
154
|
const textNode = node;
|
|
133
155
|
if (textNode.value) {
|
|
134
156
|
if (textNode.excludedFromMarkdown) return;
|
|
157
|
+
if (state.preFencePending) return;
|
|
135
158
|
if (textNode.value === " " && lastChar === "\n") return;
|
|
136
159
|
if (shouldAddSpacingBeforeText(lastChar, lastNode, textNode)) textNode.value = ` ${textNode.value}`;
|
|
137
160
|
if ((state.depthMap[34] || 0) > 0 && (state.depthMap[25] || 0) > 0) {
|
|
@@ -197,7 +220,7 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
|
|
|
197
220
|
state.lastTextNode = void 0;
|
|
198
221
|
}
|
|
199
222
|
}
|
|
200
|
-
if (output[0]?.[0] && eventType === 0 && lastChar && needsSpacing(lastChar, output[0][0], state)) {
|
|
223
|
+
if (output[0]?.[0] && eventType === 0 && !node.tagHandler?.literalEnter && lastChar && needsSpacing(lastChar, output[0][0], state)) {
|
|
201
224
|
state.buffer.push(" ");
|
|
202
225
|
state.lastContentCache = " ";
|
|
203
226
|
}
|
|
@@ -282,4 +305,4 @@ function resolvePlugins(options, hooks) {
|
|
|
282
305
|
frontmatterCallback
|
|
283
306
|
};
|
|
284
307
|
}
|
|
285
|
-
export { createMarkdownProcessor
|
|
308
|
+
export { createMarkdownProcessor, processPluginsForEvent, resolvePlugins };
|
package/dist/_chunks/src.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import "./const.mjs";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { buildTagOverrideHandlers, parseHtmlStream } from "./parse.mjs";
|
|
3
|
+
import { createMarkdownProcessor, processPluginsForEvent, resolvePlugins } from "./resolve-plugins.mjs";
|
|
4
4
|
import "./plugins.mjs";
|
|
5
5
|
import "../preset/minimal.mjs";
|
|
6
6
|
function resolveClean(clean) {
|
|
@@ -343,4 +343,4 @@ function streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
|
343
343
|
const { plugins } = resolvePlugins(options, resolveHooks(options));
|
|
344
344
|
return streamHtmlToMarkdown$1(htmlStream, options, plugins, options.plugins?.tagOverrides ? buildTagOverrideHandlers(options.plugins.tagOverrides) : void 0);
|
|
345
345
|
}
|
|
346
|
-
export {
|
|
346
|
+
export { htmlToMarkdown, streamHtmlToMarkdown };
|
package/dist/_chunks/types.d.mts
CHANGED
|
@@ -251,6 +251,12 @@ interface MdreamRuntimeState extends Partial<MdreamProcessingState> {
|
|
|
251
251
|
*/
|
|
252
252
|
listIndent?: string;
|
|
253
253
|
listIndentWidths?: number[];
|
|
254
|
+
/**
|
|
255
|
+
* <pre> fenced-code deferral (issue #97). See MarkdownState for semantics.
|
|
256
|
+
*/
|
|
257
|
+
preFencePending?: boolean;
|
|
258
|
+
preFenceLang?: string;
|
|
259
|
+
preOwnFence?: boolean;
|
|
254
260
|
}
|
|
255
261
|
type NodeEventEnter = 0;
|
|
256
262
|
type NodeEventExit = 1;
|
|
@@ -289,6 +295,12 @@ interface TagHandler {
|
|
|
289
295
|
isInline?: boolean;
|
|
290
296
|
spacing?: readonly [number, number];
|
|
291
297
|
excludesTextNodes?: boolean;
|
|
298
|
+
/**
|
|
299
|
+
* When true, the `enter` string is emitted verbatim without synthesizing a
|
|
300
|
+
* separating space before it. Set for user-supplied tagOverride enter
|
|
301
|
+
* strings so markers like `^`/`~` attach to adjacent content (issue #93).
|
|
302
|
+
*/
|
|
303
|
+
literalEnter?: boolean;
|
|
292
304
|
}
|
|
293
305
|
interface TailwindContext {
|
|
294
306
|
hidden?: boolean;
|
|
@@ -390,4 +402,4 @@ interface SplitterOptions extends EngineOptions {
|
|
|
390
402
|
*/
|
|
391
403
|
keepSeparator?: boolean;
|
|
392
404
|
}
|
|
393
|
-
export {
|
|
405
|
+
export { BuiltinPlugins, CleanOptions, ELEMENT_NODE, ElementNode, EngineOptions, ExtractedElement, FrontmatterConfig, MarkdownChunk, MdreamOptions, MdreamRuntimeState, Node, NodeEvent, NodeEventEnter$1 as NodeEventEnter, NodeEventExit$1 as NodeEventExit, PluginContext, SplitterOptions, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, TagHandler, TagOverride, TextNode, TransformPlugin };
|
package/dist/cli.mjs
CHANGED
package/dist/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { BuiltinPlugins, CleanOptions, ELEMENT_NODE, ElementNode, EngineOptions, ExtractedElement, FrontmatterConfig, MarkdownChunk, MdreamOptions, Node, NodeEvent, NodeEventEnter, NodeEventExit, PluginContext, SplitterOptions, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, TagOverride, TextNode, TransformPlugin } from "./_chunks/types.mjs";
|
|
2
|
+
import { createPlugin, htmlToMarkdown, streamHtmlToMarkdown } from "./_chunks/index.mjs";
|
|
3
|
+
import { withMinimalPreset } from "./_chunks/minimal.mjs";
|
|
4
4
|
export { type BuiltinPlugins, type CleanOptions, ELEMENT_NODE, type ElementNode, type EngineOptions, type ExtractedElement, type FrontmatterConfig, type MarkdownChunk, type MdreamOptions, type Node, type NodeEvent, NodeEventEnter, NodeEventExit, type PluginContext, type SplitterOptions, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, type TagOverride, type TextNode, type TransformPlugin, createPlugin, htmlToMarkdown, streamHtmlToMarkdown, withMinimalPreset };
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { htmlToMarkdown, streamHtmlToMarkdown } from "./_chunks/src.mjs";
|
|
2
|
+
import { ELEMENT_NODE, NodeEventEnter, NodeEventExit, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE } from "./_chunks/const.mjs";
|
|
3
|
+
import { createPlugin } from "./_chunks/plugins.mjs";
|
|
4
4
|
import { withMinimalPreset } from "./preset/minimal.mjs";
|
|
5
5
|
export { ELEMENT_NODE, NodeEventEnter, NodeEventExit, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, createPlugin, htmlToMarkdown, streamHtmlToMarkdown, withMinimalPreset };
|
package/dist/parse.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ElementNode, Node, NodeEvent, TagHandler, TransformPlugin } from "./_chunks/types.mjs";
|
|
2
2
|
interface ParseOptions {
|
|
3
3
|
resolvedPlugins?: TransformPlugin[];
|
|
4
4
|
}
|
|
@@ -27,6 +27,13 @@ interface ParseState {
|
|
|
27
27
|
inSingleQuote?: boolean;
|
|
28
28
|
inDoubleQuote?: boolean;
|
|
29
29
|
inBacktick?: boolean;
|
|
30
|
+
/**
|
|
31
|
+
* Whether the current non-nesting tag is quote-aware rawtext (script/style only).
|
|
32
|
+
* For these, a `</script>`/`</style>` inside a JS/CSS string literal must not close
|
|
33
|
+
* the tag. Other non-nesting tags (template, textarea, ...) hold literal HTML/text
|
|
34
|
+
* where apostrophes are not string delimiters, so their closing tag always wins.
|
|
35
|
+
*/
|
|
36
|
+
inRawTextQuoteAware?: boolean;
|
|
30
37
|
/** Backslash escaping state tracking - avoids checking previous character */
|
|
31
38
|
lastCharWasBackslash?: boolean;
|
|
32
39
|
/** Resolved plugin instances for event processing */
|
package/dist/parse.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { parseAttributes, parseHtml, parseHtmlStream } from "./_chunks/parse.mjs";
|
|
2
2
|
export { parseAttributes, parseHtml, parseHtmlStream };
|
package/dist/plugins.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { ElementNode, ExtractedElement as ExtractedElement$1, MdreamRuntimeState, TransformPlugin } from "./_chunks/types.mjs";
|
|
2
|
+
import { createPlugin } from "./_chunks/index.mjs";
|
|
3
3
|
interface ExtractedElement extends ElementNode {
|
|
4
4
|
textContent: string;
|
|
5
5
|
}
|
package/dist/plugins.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createPlugin, extractionCollectorPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin } from "./_chunks/plugins.mjs";
|
|
2
2
|
export { createPlugin, extractionCollectorPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin };
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { withMinimalPreset } from "../_chunks/minimal.mjs";
|
|
2
2
|
export { withMinimalPreset };
|
package/dist/splitter.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MarkdownChunk, SplitterOptions } from "./_chunks/types.mjs";
|
|
2
2
|
/**
|
|
3
3
|
* Convert HTML to Markdown and split into chunks in single pass.
|
|
4
4
|
* Yields chunks during HTML event processing for better memory efficiency.
|
package/dist/splitter.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import "./_chunks/const.mjs";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { parseHtmlStream } from "./_chunks/parse.mjs";
|
|
3
|
+
import { createMarkdownProcessor, processPluginsForEvent, resolvePlugins } from "./_chunks/resolve-plugins.mjs";
|
|
4
4
|
const MARKDOWN_HEADER_LINE_RE = /^#{1,6}\s+/;
|
|
5
5
|
const NEWLINE_RE = /\n/g;
|
|
6
6
|
const DEFAULT_HEADERS_TO_SPLIT_ON = [
|
package/package.json
CHANGED