@mdream/js 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/const.mjs +3 -2
- package/dist/_chunks/index.d.mts +2 -2
- package/dist/_chunks/minimal.d.mts +2 -2
- package/dist/_chunks/parse.mjs +112 -12
- package/dist/_chunks/plugins.mjs +18 -10
- package/dist/_chunks/resolve-plugins.mjs +28 -5
- package/dist/_chunks/src.mjs +3 -3
- package/dist/_chunks/types.d.mts +7 -1
- package/dist/cli.mjs +1 -1
- package/dist/index.d.mts +3 -3
- package/dist/index.mjs +3 -3
- package/dist/parse.d.mts +8 -1
- package/dist/parse.mjs +1 -1
- package/dist/plugins.d.mts +2 -2
- package/dist/plugins.mjs +1 -1
- package/dist/preset/minimal.d.mts +1 -1
- package/dist/splitter.d.mts +1 -1
- package/dist/splitter.mjs +2 -2
- package/package.json +1 -1
package/dist/_chunks/const.mjs
CHANGED
|
@@ -116,7 +116,8 @@ const TagIdMap = {
|
|
|
116
116
|
main: 104,
|
|
117
117
|
header: 105,
|
|
118
118
|
figcaption: 106,
|
|
119
|
-
caption: 107
|
|
119
|
+
caption: 107,
|
|
120
|
+
datalist: 108
|
|
120
121
|
};
|
|
121
122
|
Object.entries(TagIdMap).map(([name, id]) => [id, name]);
|
|
122
123
|
const NO_SPACING = [0, 0];
|
|
@@ -124,4 +125,4 @@ const DEFAULT_BLOCK_SPACING = [2, 2];
|
|
|
124
125
|
const BLOCKQUOTE_SPACING = [1, 1];
|
|
125
126
|
const LIST_ITEM_SPACING = [1, 0];
|
|
126
127
|
const TABLE_ROW_SPACING = [0, 1];
|
|
127
|
-
export {
|
|
128
|
+
export { BLOCKQUOTE_SPACING, DEFAULT_BLOCK_SPACING, ELEMENT_NODE, LIST_ITEM_SPACING, NO_SPACING, NodeEventEnter, NodeEventExit, TABLE_ROW_SPACING, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, TagIdMap };
|
package/dist/_chunks/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MdreamOptions, TransformPlugin } from "./types.mjs";
|
|
2
2
|
/**
|
|
3
3
|
* Create a plugin with type-safe hook definitions.
|
|
4
4
|
* All TransformPlugin fields are optional, so this is a typed identity function.
|
|
@@ -6,4 +6,4 @@ import { _ as TransformPlugin, c as MdreamOptions } from "./types.mjs";
|
|
|
6
6
|
declare function createPlugin(plugin: TransformPlugin): TransformPlugin;
|
|
7
7
|
declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
|
|
8
8
|
declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
9
|
-
export {
|
|
9
|
+
export { createPlugin, htmlToMarkdown, streamHtmlToMarkdown };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { EngineOptions } from "./types.mjs";
|
|
2
2
|
/**
|
|
3
3
|
* Creates a configurable minimal preset with advanced options.
|
|
4
4
|
* Returns declarative plugin config that works with both JS and Rust engines.
|
|
5
5
|
*/
|
|
6
6
|
declare function withMinimalPreset<T extends EngineOptions>(options?: T): T;
|
|
7
|
-
export { withMinimalPreset
|
|
7
|
+
export { withMinimalPreset };
|
package/dist/_chunks/parse.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { BLOCKQUOTE_SPACING, LIST_ITEM_SPACING, NO_SPACING, TABLE_ROW_SPACING, TagIdMap } from "./const.mjs";
|
|
2
2
|
function resolveUrl(url, origin) {
|
|
3
3
|
if (!url) return url;
|
|
4
4
|
if (url.startsWith("//")) return `https:${url}`;
|
|
@@ -155,9 +155,29 @@ const tagHandlers = {
|
|
|
155
155
|
},
|
|
156
156
|
spacing: BLOCKQUOTE_SPACING
|
|
157
157
|
},
|
|
158
|
+
[34]: {
|
|
159
|
+
enter: ({ node, state }) => {
|
|
160
|
+
state.preFencePending = true;
|
|
161
|
+
state.preOwnFence = false;
|
|
162
|
+
state.preFenceLang = getLanguageFromClass(node.attributes?.class);
|
|
163
|
+
},
|
|
164
|
+
exit: ({ node, state }) => {
|
|
165
|
+
const ownFence = state.preOwnFence;
|
|
166
|
+
state.preFencePending = false;
|
|
167
|
+
state.preOwnFence = false;
|
|
168
|
+
if (!ownFence) return;
|
|
169
|
+
if ((node.depthMap[25] || 0) > 0) {
|
|
170
|
+
const indent = state.listIndent;
|
|
171
|
+
return `\n${indent}\`\`\`\n\n${indent}`;
|
|
172
|
+
}
|
|
173
|
+
return `\n\`\`\``;
|
|
174
|
+
}
|
|
175
|
+
},
|
|
158
176
|
[23]: {
|
|
159
177
|
enter: ({ node, state }) => {
|
|
160
178
|
if ((node.depthMap[34] || 0) > 0) {
|
|
179
|
+
if (state.preOwnFence) return;
|
|
180
|
+
state.preFencePending = false;
|
|
161
181
|
const language = getLanguageFromClass(node.attributes?.class);
|
|
162
182
|
if ((node.depthMap[25] || 0) > 0) return `\n\n${state.listIndent}\`\`\`${language}\n`;
|
|
163
183
|
return `\`\`\`${language}\n`;
|
|
@@ -171,6 +191,7 @@ const tagHandlers = {
|
|
|
171
191
|
},
|
|
172
192
|
exit: ({ node, state }) => {
|
|
173
193
|
if ((node.depthMap[34] || 0) > 0) {
|
|
194
|
+
if (state.preOwnFence) return;
|
|
174
195
|
if ((node.depthMap[25] || 0) > 0) {
|
|
175
196
|
const indent = state.listIndent;
|
|
176
197
|
return `\n${indent}\`\`\`\n\n${indent}`;
|
|
@@ -449,7 +470,11 @@ const tagHandlers = {
|
|
|
449
470
|
[75]: { spacing: NO_SPACING },
|
|
450
471
|
[76]: { spacing: NO_SPACING },
|
|
451
472
|
[77]: { spacing: NO_SPACING },
|
|
452
|
-
[78]: {
|
|
473
|
+
[78]: {
|
|
474
|
+
isNonNesting: true,
|
|
475
|
+
excludesTextNodes: true,
|
|
476
|
+
spacing: NO_SPACING
|
|
477
|
+
},
|
|
453
478
|
[79]: {
|
|
454
479
|
enter: () => "",
|
|
455
480
|
exit: () => "",
|
|
@@ -489,6 +514,11 @@ const tagHandlers = {
|
|
|
489
514
|
excludesTextNodes: true,
|
|
490
515
|
spacing: NO_SPACING
|
|
491
516
|
},
|
|
517
|
+
[108]: {
|
|
518
|
+
isNonNesting: true,
|
|
519
|
+
excludesTextNodes: true,
|
|
520
|
+
spacing: NO_SPACING
|
|
521
|
+
},
|
|
492
522
|
[85]: {
|
|
493
523
|
isNonNesting: true,
|
|
494
524
|
spacing: NO_SPACING
|
|
@@ -964,7 +994,7 @@ function parseHtml(html, options = {}) {
|
|
|
964
994
|
return {
|
|
965
995
|
events,
|
|
966
996
|
remainingHtml: parseHtmlInternal(html, {
|
|
967
|
-
depthMap: new Uint8Array(
|
|
997
|
+
depthMap: new Uint8Array(109),
|
|
968
998
|
depth: 0,
|
|
969
999
|
resolvedPlugins: options.resolvedPlugins || []
|
|
970
1000
|
}, (event) => {
|
|
@@ -977,7 +1007,7 @@ function parseHtmlStream(html, state, onEvent) {
|
|
|
977
1007
|
}
|
|
978
1008
|
function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
979
1009
|
let textBuffer = "";
|
|
980
|
-
state.depthMap ??= new Uint8Array(
|
|
1010
|
+
state.depthMap ??= new Uint8Array(109);
|
|
981
1011
|
state.depth ??= 0;
|
|
982
1012
|
state.lastCharWasWhitespace ??= true;
|
|
983
1013
|
state.justClosedTag ??= false;
|
|
@@ -1014,7 +1044,7 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1014
1044
|
else if (currentCharCode === CLOSE_BRACKET_CHAR && state.depthMap[26]) textBuffer += "\\]";
|
|
1015
1045
|
else if (currentCharCode === GT_CHAR && state.depthMap[22]) textBuffer += "\\>";
|
|
1016
1046
|
else textBuffer += htmlChunk[i];
|
|
1017
|
-
if (state.
|
|
1047
|
+
if (state.inRawTextQuoteAware) {
|
|
1018
1048
|
if (!state.lastCharWasBackslash) {
|
|
1019
1049
|
if (currentCharCode === APOS_CHAR && !state.inDoubleQuote && !state.inBacktick) state.inSingleQuote = !state.inSingleQuote;
|
|
1020
1050
|
else if (currentCharCode === QUOTE_CHAR && !state.inSingleQuote && !state.inBacktick) state.inDoubleQuote = !state.inDoubleQuote;
|
|
@@ -1032,6 +1062,26 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1032
1062
|
}
|
|
1033
1063
|
const nextCharCode = htmlChunk.charCodeAt(i + 1);
|
|
1034
1064
|
if (nextCharCode === EXCLAMATION_CHAR) {
|
|
1065
|
+
if (htmlChunk.charCodeAt(i + 2) === OPEN_BRACKET_CHAR) {
|
|
1066
|
+
if (htmlChunk.startsWith("<![CDATA[", i)) {
|
|
1067
|
+
const end = htmlChunk.indexOf("]]>", i + 9);
|
|
1068
|
+
if (end === -1) {
|
|
1069
|
+
textBuffer += htmlChunk.substring(i);
|
|
1070
|
+
break;
|
|
1071
|
+
}
|
|
1072
|
+
if (textBuffer.length > 0) {
|
|
1073
|
+
processTextBuffer(textBuffer, state, handleEvent);
|
|
1074
|
+
textBuffer = "";
|
|
1075
|
+
}
|
|
1076
|
+
processCdataSection(htmlChunk.substring(i + 9, end), state, handleEvent);
|
|
1077
|
+
i = end + 3;
|
|
1078
|
+
continue;
|
|
1079
|
+
}
|
|
1080
|
+
if (chunkLength - i < 9 && "<![CDATA[".startsWith(htmlChunk.substring(i))) {
|
|
1081
|
+
textBuffer += htmlChunk.substring(i);
|
|
1082
|
+
break;
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1035
1085
|
if (textBuffer.length > 0) {
|
|
1036
1086
|
processTextBuffer(textBuffer, state, handleEvent);
|
|
1037
1087
|
textBuffer = "";
|
|
@@ -1044,7 +1094,7 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1044
1094
|
}
|
|
1045
1095
|
} else if (nextCharCode === SLASH_CHAR) {
|
|
1046
1096
|
if (state.currentNode?.tagHandler?.isNonNesting) {
|
|
1047
|
-
if (state.inSingleQuote || state.inDoubleQuote || state.inBacktick) {
|
|
1097
|
+
if (state.inRawTextQuoteAware && (state.inSingleQuote || state.inDoubleQuote || state.inBacktick)) {
|
|
1048
1098
|
textBuffer += htmlChunk[i];
|
|
1049
1099
|
i++;
|
|
1050
1100
|
continue;
|
|
@@ -1090,10 +1140,8 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1090
1140
|
const tagId = TagIdMap[tagName] ?? -1;
|
|
1091
1141
|
i2 = tagNameEnd;
|
|
1092
1142
|
if (state.currentNode?.tagHandler?.isNonNesting) {
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
continue;
|
|
1096
|
-
}
|
|
1143
|
+
textBuffer += htmlChunk[i++];
|
|
1144
|
+
continue;
|
|
1097
1145
|
}
|
|
1098
1146
|
if (!tagName) {
|
|
1099
1147
|
textBuffer += htmlChunk[i++];
|
|
@@ -1114,6 +1162,10 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1114
1162
|
}
|
|
1115
1163
|
}
|
|
1116
1164
|
}
|
|
1165
|
+
state.inSingleQuote = false;
|
|
1166
|
+
state.inDoubleQuote = false;
|
|
1167
|
+
state.inBacktick = false;
|
|
1168
|
+
state.lastCharWasBackslash = false;
|
|
1117
1169
|
return textBuffer;
|
|
1118
1170
|
}
|
|
1119
1171
|
function processTextBuffer(textBuffer, state, handleEvent) {
|
|
@@ -1121,7 +1173,30 @@ function processTextBuffer(textBuffer, state, handleEvent) {
|
|
|
1121
1173
|
const containsWhitespace = state.textBufferContainsWhitespace;
|
|
1122
1174
|
state.textBufferContainsNonWhitespace = false;
|
|
1123
1175
|
state.textBufferContainsWhitespace = false;
|
|
1124
|
-
if (!state.currentNode)
|
|
1176
|
+
if (!state.currentNode) {
|
|
1177
|
+
if (!containsNonWhitespace) return;
|
|
1178
|
+
let rootText = textBuffer;
|
|
1179
|
+
if (rootText.length === 0) return;
|
|
1180
|
+
if (state.hasEncodedHtmlEntity) {
|
|
1181
|
+
rootText = decodeHTMLEntities(String(rootText));
|
|
1182
|
+
state.hasEncodedHtmlEntity = false;
|
|
1183
|
+
}
|
|
1184
|
+
const rootTextNode = {
|
|
1185
|
+
type: 2,
|
|
1186
|
+
value: rootText,
|
|
1187
|
+
parent: null,
|
|
1188
|
+
index: 0,
|
|
1189
|
+
depth: state.depth,
|
|
1190
|
+
containsWhitespace,
|
|
1191
|
+
excludedFromMarkdown: false
|
|
1192
|
+
};
|
|
1193
|
+
handleEvent({
|
|
1194
|
+
type: 0,
|
|
1195
|
+
node: rootTextNode
|
|
1196
|
+
});
|
|
1197
|
+
state.lastTextNode = rootTextNode;
|
|
1198
|
+
return;
|
|
1199
|
+
}
|
|
1125
1200
|
const excludesTextNodes = state.currentNode?.tagHandler?.excludesTextNodes;
|
|
1126
1201
|
const inPreTag = (state.depthMap[34] || 0) > 0;
|
|
1127
1202
|
if (!inPreTag && !containsNonWhitespace && !state.currentNode.childTextNodeIndex) return;
|
|
@@ -1219,6 +1294,7 @@ function closeNode(node, state, handleEvent) {
|
|
|
1219
1294
|
state.inDoubleQuote = false;
|
|
1220
1295
|
state.inBacktick = false;
|
|
1221
1296
|
state.lastCharWasBackslash = false;
|
|
1297
|
+
state.inRawTextQuoteAware = false;
|
|
1222
1298
|
}
|
|
1223
1299
|
state.depth--;
|
|
1224
1300
|
handleEvent({
|
|
@@ -1270,6 +1346,29 @@ function processCommentOrDoctype(htmlChunk, position) {
|
|
|
1270
1346
|
};
|
|
1271
1347
|
}
|
|
1272
1348
|
}
|
|
1349
|
+
function processCdataSection(content, state, handleEvent) {
|
|
1350
|
+
if (!state.tagOverrideHandlers?.has("#cdata-section")) return;
|
|
1351
|
+
const open = processOpeningTag("#cdata-section", -1, ">", 0, state, handleEvent);
|
|
1352
|
+
if (!open.complete || open.selfClosing) return;
|
|
1353
|
+
const node = state.currentNode;
|
|
1354
|
+
if (content.length > 0 && !node.tagHandler?.excludesTextNodes) {
|
|
1355
|
+
const textNode = {
|
|
1356
|
+
type: 2,
|
|
1357
|
+
value: content,
|
|
1358
|
+
parent: node,
|
|
1359
|
+
index: node.currentWalkIndex++,
|
|
1360
|
+
depth: state.depth,
|
|
1361
|
+
containsWhitespace: false
|
|
1362
|
+
};
|
|
1363
|
+
for (const parent of traverseUpToFirstBlockNode(node)) parent.childTextNodeIndex = (parent.childTextNodeIndex || 0) + 1;
|
|
1364
|
+
handleEvent({
|
|
1365
|
+
type: 0,
|
|
1366
|
+
node: textNode
|
|
1367
|
+
});
|
|
1368
|
+
state.lastTextNode = textNode;
|
|
1369
|
+
}
|
|
1370
|
+
closeNode(state.currentNode ?? null, state, handleEvent);
|
|
1371
|
+
}
|
|
1273
1372
|
function processOpeningTag(tagName, tagId, htmlChunk, i, state, handleEvent) {
|
|
1274
1373
|
if (state.currentNode?.tagHandler?.isNonNesting) closeNode(state.currentNode, state, handleEvent);
|
|
1275
1374
|
const tagHandler = state.tagOverrideHandlers?.get(tagName) ?? tagHandlers[tagId];
|
|
@@ -1311,6 +1410,7 @@ function processOpeningTag(tagName, tagId, htmlChunk, i, state, handleEvent) {
|
|
|
1311
1410
|
state.inDoubleQuote = false;
|
|
1312
1411
|
state.inBacktick = false;
|
|
1313
1412
|
state.lastCharWasBackslash = false;
|
|
1413
|
+
state.inRawTextQuoteAware = tag.tagId === 52 || tag.tagId === 53;
|
|
1314
1414
|
}
|
|
1315
1415
|
if (result.selfClosing) {
|
|
1316
1416
|
closeNode(tag, state, handleEvent);
|
|
@@ -1454,4 +1554,4 @@ function parseAttributes(attrStr) {
|
|
|
1454
1554
|
}
|
|
1455
1555
|
return result;
|
|
1456
1556
|
}
|
|
1457
|
-
export { buildTagOverrideHandlers
|
|
1557
|
+
export { buildTagOverrideHandlers, parseAttributes, parseHtml, parseHtmlStream };
|
package/dist/_chunks/plugins.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { TagIdMap } from "./const.mjs";
|
|
2
2
|
function createPlugin(plugin) {
|
|
3
3
|
return plugin;
|
|
4
4
|
}
|
|
@@ -216,14 +216,23 @@ function compileSelector(selector) {
|
|
|
216
216
|
};
|
|
217
217
|
return parseSelector(selector);
|
|
218
218
|
}
|
|
219
|
+
function isHidden(element) {
|
|
220
|
+
const style = element.attributes?.style;
|
|
221
|
+
if (style && (style.includes("display:none") || style.includes("display: none") || style.includes("visibility:hidden") || style.includes("visibility: hidden") || style.includes("position:absolute") || style.includes("position: absolute") || style.includes("position:fixed") || style.includes("position: fixed"))) return true;
|
|
222
|
+
const hidden = element.attributes?.hidden;
|
|
223
|
+
return hidden !== void 0 && hidden.toLowerCase() !== "until-found";
|
|
224
|
+
}
|
|
219
225
|
function filterPlugin(options = {}) {
|
|
220
226
|
const includeSelectors = options.include?.map((selector) => compileSelector(selector)) || [];
|
|
221
227
|
const excludeSelectors = options.exclude?.map((selector) => compileSelector(selector)) || [];
|
|
222
228
|
const processChildren = options.processChildren !== false;
|
|
229
|
+
const hiddenNodes = /* @__PURE__ */ new WeakSet();
|
|
223
230
|
return createPlugin({ beforeNodeProcess(event) {
|
|
224
231
|
const { node } = event;
|
|
225
232
|
if (node.type === 2) {
|
|
226
|
-
|
|
233
|
+
const parent = node.parent;
|
|
234
|
+
if (parent && hiddenNodes.has(parent)) return { skip: true };
|
|
235
|
+
let currentParent = parent;
|
|
227
236
|
while (currentParent && excludeSelectors.length) {
|
|
228
237
|
if (excludeSelectors.some((selector) => selector.matches(currentParent))) return { skip: true };
|
|
229
238
|
currentParent = currentParent.parent;
|
|
@@ -232,15 +241,14 @@ function filterPlugin(options = {}) {
|
|
|
232
241
|
}
|
|
233
242
|
if (node.type !== 1) return;
|
|
234
243
|
const element = node;
|
|
235
|
-
if (
|
|
236
|
-
|
|
237
|
-
|
|
244
|
+
if ((element.parent ? hiddenNodes.has(element.parent) : false) || isHidden(element)) {
|
|
245
|
+
hiddenNodes.add(element);
|
|
246
|
+
return { skip: true };
|
|
238
247
|
}
|
|
248
|
+
if (excludeSelectors.length && excludeSelectors.some((selector) => selector.matches(element))) return { skip: true };
|
|
239
249
|
let currentParent = element.parent;
|
|
240
|
-
while (currentParent) {
|
|
241
|
-
if (excludeSelectors.
|
|
242
|
-
if (excludeSelectors.some((selector) => selector.matches(currentParent))) return { skip: true };
|
|
243
|
-
}
|
|
250
|
+
while (currentParent && excludeSelectors.length) {
|
|
251
|
+
if (excludeSelectors.some((selector) => selector.matches(currentParent))) return { skip: true };
|
|
244
252
|
currentParent = currentParent.parent;
|
|
245
253
|
}
|
|
246
254
|
if (includeSelectors.length) {
|
|
@@ -660,4 +668,4 @@ function tailwindPlugin() {
|
|
|
660
668
|
}
|
|
661
669
|
});
|
|
662
670
|
}
|
|
663
|
-
export {
|
|
671
|
+
export { createPlugin, extractionCollectorPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { DEFAULT_BLOCK_SPACING, NO_SPACING } from "./const.mjs";
|
|
2
|
+
import { parseHtmlStream } from "./parse.mjs";
|
|
3
|
+
import { extractionCollectorPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin } from "./plugins.mjs";
|
|
4
4
|
function processPluginsForEvent(event, plugins, state, processEvent) {
|
|
5
5
|
if (plugins?.length) {
|
|
6
6
|
if (event.node.type === 1 && event.type === 0) {
|
|
@@ -105,11 +105,26 @@ function calculateNewLineConfig(node) {
|
|
|
105
105
|
if (tagId === -1) return NO_SPACING;
|
|
106
106
|
return DEFAULT_BLOCK_SPACING;
|
|
107
107
|
}
|
|
108
|
+
function hasNonWhitespace(value) {
|
|
109
|
+
for (let i = 0; i < value.length; i++) {
|
|
110
|
+
const c = value.charCodeAt(i);
|
|
111
|
+
if (c !== 32 && c !== 9 && c !== 10 && c !== 13) return true;
|
|
112
|
+
}
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
function flushPreFence(state) {
|
|
116
|
+
state.preFencePending = false;
|
|
117
|
+
state.preOwnFence = true;
|
|
118
|
+
const lang = state.preFenceLang || "";
|
|
119
|
+
const fence = (state.depthMap[25] || 0) > 0 ? `\n\n${state.listIndent}\`\`\`${lang}\n${state.listIndent}` : `\`\`\`${lang}\n`;
|
|
120
|
+
state.buffer.push(fence);
|
|
121
|
+
state.lastContentCache = fence;
|
|
122
|
+
}
|
|
108
123
|
function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverrideHandlers) {
|
|
109
124
|
const state = {
|
|
110
125
|
options,
|
|
111
126
|
buffer: [],
|
|
112
|
-
depthMap: new Uint8Array(
|
|
127
|
+
depthMap: new Uint8Array(109),
|
|
113
128
|
listIndent: "",
|
|
114
129
|
listIndentWidths: []
|
|
115
130
|
};
|
|
@@ -120,6 +135,13 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
|
|
|
120
135
|
state.lastNode = event.node;
|
|
121
136
|
state.depth = node.depth;
|
|
122
137
|
const buff = state.buffer;
|
|
138
|
+
if (state.preFencePending && eventType === 0) {
|
|
139
|
+
if (node.type === 1) {
|
|
140
|
+
const el = node;
|
|
141
|
+
if (el.tagId === 23 && el.parent?.tagId === 34) state.preFencePending = false;
|
|
142
|
+
else if (el.tagId !== 34) flushPreFence(state);
|
|
143
|
+
} else if (node.type === 2 && hasNonWhitespace(node.value)) flushPreFence(state);
|
|
144
|
+
}
|
|
123
145
|
const lastBuffEntry = buff.at(-1);
|
|
124
146
|
const lastChar = lastBuffEntry?.charAt(lastBuffEntry.length - 1) || "";
|
|
125
147
|
let secondLastChar;
|
|
@@ -132,6 +154,7 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
|
|
|
132
154
|
const textNode = node;
|
|
133
155
|
if (textNode.value) {
|
|
134
156
|
if (textNode.excludedFromMarkdown) return;
|
|
157
|
+
if (state.preFencePending) return;
|
|
135
158
|
if (textNode.value === " " && lastChar === "\n") return;
|
|
136
159
|
if (shouldAddSpacingBeforeText(lastChar, lastNode, textNode)) textNode.value = ` ${textNode.value}`;
|
|
137
160
|
if ((state.depthMap[34] || 0) > 0 && (state.depthMap[25] || 0) > 0) {
|
|
@@ -282,4 +305,4 @@ function resolvePlugins(options, hooks) {
|
|
|
282
305
|
frontmatterCallback
|
|
283
306
|
};
|
|
284
307
|
}
|
|
285
|
-
export { createMarkdownProcessor
|
|
308
|
+
export { createMarkdownProcessor, processPluginsForEvent, resolvePlugins };
|
package/dist/_chunks/src.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import "./const.mjs";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { buildTagOverrideHandlers, parseHtmlStream } from "./parse.mjs";
|
|
3
|
+
import { createMarkdownProcessor, processPluginsForEvent, resolvePlugins } from "./resolve-plugins.mjs";
|
|
4
4
|
import "./plugins.mjs";
|
|
5
5
|
import "../preset/minimal.mjs";
|
|
6
6
|
function resolveClean(clean) {
|
|
@@ -343,4 +343,4 @@ function streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
|
343
343
|
const { plugins } = resolvePlugins(options, resolveHooks(options));
|
|
344
344
|
return streamHtmlToMarkdown$1(htmlStream, options, plugins, options.plugins?.tagOverrides ? buildTagOverrideHandlers(options.plugins.tagOverrides) : void 0);
|
|
345
345
|
}
|
|
346
|
-
export {
|
|
346
|
+
export { htmlToMarkdown, streamHtmlToMarkdown };
|
package/dist/_chunks/types.d.mts
CHANGED
|
@@ -251,6 +251,12 @@ interface MdreamRuntimeState extends Partial<MdreamProcessingState> {
|
|
|
251
251
|
*/
|
|
252
252
|
listIndent?: string;
|
|
253
253
|
listIndentWidths?: number[];
|
|
254
|
+
/**
|
|
255
|
+
* <pre> fenced-code deferral (issue #97). See MarkdownState for semantics.
|
|
256
|
+
*/
|
|
257
|
+
preFencePending?: boolean;
|
|
258
|
+
preFenceLang?: string;
|
|
259
|
+
preOwnFence?: boolean;
|
|
254
260
|
}
|
|
255
261
|
type NodeEventEnter = 0;
|
|
256
262
|
type NodeEventExit = 1;
|
|
@@ -396,4 +402,4 @@ interface SplitterOptions extends EngineOptions {
|
|
|
396
402
|
*/
|
|
397
403
|
keepSeparator?: boolean;
|
|
398
404
|
}
|
|
399
|
-
export {
|
|
405
|
+
export { BuiltinPlugins, CleanOptions, ELEMENT_NODE, ElementNode, EngineOptions, ExtractedElement, FrontmatterConfig, MarkdownChunk, MdreamOptions, MdreamRuntimeState, Node, NodeEvent, NodeEventEnter$1 as NodeEventEnter, NodeEventExit$1 as NodeEventExit, PluginContext, SplitterOptions, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, TagHandler, TagOverride, TextNode, TransformPlugin };
|
package/dist/cli.mjs
CHANGED
package/dist/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { BuiltinPlugins, CleanOptions, ELEMENT_NODE, ElementNode, EngineOptions, ExtractedElement, FrontmatterConfig, MarkdownChunk, MdreamOptions, Node, NodeEvent, NodeEventEnter, NodeEventExit, PluginContext, SplitterOptions, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, TagOverride, TextNode, TransformPlugin } from "./_chunks/types.mjs";
|
|
2
|
+
import { createPlugin, htmlToMarkdown, streamHtmlToMarkdown } from "./_chunks/index.mjs";
|
|
3
|
+
import { withMinimalPreset } from "./_chunks/minimal.mjs";
|
|
4
4
|
export { type BuiltinPlugins, type CleanOptions, ELEMENT_NODE, type ElementNode, type EngineOptions, type ExtractedElement, type FrontmatterConfig, type MarkdownChunk, type MdreamOptions, type Node, type NodeEvent, NodeEventEnter, NodeEventExit, type PluginContext, type SplitterOptions, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, type TagOverride, type TextNode, type TransformPlugin, createPlugin, htmlToMarkdown, streamHtmlToMarkdown, withMinimalPreset };
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { htmlToMarkdown, streamHtmlToMarkdown } from "./_chunks/src.mjs";
|
|
2
|
+
import { ELEMENT_NODE, NodeEventEnter, NodeEventExit, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE } from "./_chunks/const.mjs";
|
|
3
|
+
import { createPlugin } from "./_chunks/plugins.mjs";
|
|
4
4
|
import { withMinimalPreset } from "./preset/minimal.mjs";
|
|
5
5
|
export { ELEMENT_NODE, NodeEventEnter, NodeEventExit, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, createPlugin, htmlToMarkdown, streamHtmlToMarkdown, withMinimalPreset };
|
package/dist/parse.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ElementNode, Node, NodeEvent, TagHandler, TransformPlugin } from "./_chunks/types.mjs";
|
|
2
2
|
interface ParseOptions {
|
|
3
3
|
resolvedPlugins?: TransformPlugin[];
|
|
4
4
|
}
|
|
@@ -27,6 +27,13 @@ interface ParseState {
|
|
|
27
27
|
inSingleQuote?: boolean;
|
|
28
28
|
inDoubleQuote?: boolean;
|
|
29
29
|
inBacktick?: boolean;
|
|
30
|
+
/**
|
|
31
|
+
* Whether the current non-nesting tag is quote-aware rawtext (script/style only).
|
|
32
|
+
* For these, a `</script>`/`</style>` inside a JS/CSS string literal must not close
|
|
33
|
+
* the tag. Other non-nesting tags (template, textarea, ...) hold literal HTML/text
|
|
34
|
+
* where apostrophes are not string delimiters, so their closing tag always wins.
|
|
35
|
+
*/
|
|
36
|
+
inRawTextQuoteAware?: boolean;
|
|
30
37
|
/** Backslash escaping state tracking - avoids checking previous character */
|
|
31
38
|
lastCharWasBackslash?: boolean;
|
|
32
39
|
/** Resolved plugin instances for event processing */
|
package/dist/parse.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { parseAttributes, parseHtml, parseHtmlStream } from "./_chunks/parse.mjs";
|
|
2
2
|
export { parseAttributes, parseHtml, parseHtmlStream };
|
package/dist/plugins.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { ElementNode, ExtractedElement as ExtractedElement$1, MdreamRuntimeState, TransformPlugin } from "./_chunks/types.mjs";
|
|
2
|
+
import { createPlugin } from "./_chunks/index.mjs";
|
|
3
3
|
interface ExtractedElement extends ElementNode {
|
|
4
4
|
textContent: string;
|
|
5
5
|
}
|
package/dist/plugins.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createPlugin, extractionCollectorPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin } from "./_chunks/plugins.mjs";
|
|
2
2
|
export { createPlugin, extractionCollectorPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin };
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { withMinimalPreset } from "../_chunks/minimal.mjs";
|
|
2
2
|
export { withMinimalPreset };
|
package/dist/splitter.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MarkdownChunk, SplitterOptions } from "./_chunks/types.mjs";
|
|
2
2
|
/**
|
|
3
3
|
* Convert HTML to Markdown and split into chunks in single pass.
|
|
4
4
|
* Yields chunks during HTML event processing for better memory efficiency.
|
package/dist/splitter.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import "./_chunks/const.mjs";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { parseHtmlStream } from "./_chunks/parse.mjs";
|
|
3
|
+
import { createMarkdownProcessor, processPluginsForEvent, resolvePlugins } from "./_chunks/resolve-plugins.mjs";
|
|
4
4
|
const MARKDOWN_HEADER_LINE_RE = /^#{1,6}\s+/;
|
|
5
5
|
const NEWLINE_RE = /\n/g;
|
|
6
6
|
const DEFAULT_HEADERS_TO_SPLIT_ON = [
|
package/package.json
CHANGED