@mdream/js 1.7.1 → 2.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/index.d.mts +3 -4
- package/dist/_chunks/markdown-processor.mjs +906 -131
- package/dist/_chunks/minimal.d.mts +1 -2
- package/dist/_chunks/parse.mjs +161 -68
- package/dist/_chunks/stream.mjs +4 -2
- package/dist/_chunks/types.d.mts +33 -29
- package/dist/cli.d.mts +1 -1
- package/dist/cli.mjs +3 -2
- package/dist/core.d.mts +4 -5
- package/dist/core.mjs +4 -2
- package/dist/index.mjs +54 -1
- package/dist/llms-txt.d.mts +8 -9
- package/dist/negotiate.d.mts +4 -5
- package/dist/parse.d.mts +7 -8
- package/dist/plugins.d.mts +7 -7
- package/dist/preset/minimal.mjs +20 -15
- package/dist/splitter.d.mts +3 -3
- package/package.json +1 -1
- package/dist/_chunks/src.mjs +0 -316
|
@@ -3,5 +3,4 @@ import { EngineOptions } from "./types.mjs";
|
|
|
3
3
|
* Creates a configurable minimal preset with advanced options.
|
|
4
4
|
* Returns declarative plugin config that works with both JS and Rust engines.
|
|
5
5
|
*/
|
|
6
|
-
declare function withMinimalPreset<T extends EngineOptions>(options?: T): T;
|
|
7
|
-
export { withMinimalPreset };
|
|
6
|
+
export declare function withMinimalPreset<T extends EngineOptions>(options?: T): T;
|
package/dist/_chunks/parse.mjs
CHANGED
|
@@ -274,6 +274,13 @@ function schemeMatches(href, start, end, scheme) {
|
|
|
274
274
|
}
|
|
275
275
|
return false;
|
|
276
276
|
}
|
|
277
|
+
function isDataUrl(src) {
|
|
278
|
+
let start = 0;
|
|
279
|
+
while (start < src.length && src.charCodeAt(start) <= 32) start++;
|
|
280
|
+
let end = src.length;
|
|
281
|
+
while (end > start && src.charCodeAt(end - 1) <= 32) end--;
|
|
282
|
+
return schemeMatches(src, start, end, "data:");
|
|
283
|
+
}
|
|
277
284
|
function isEmptyLinkHref(href) {
|
|
278
285
|
let start = 0;
|
|
279
286
|
while (start < href.length && href.charCodeAt(start) <= 32) start++;
|
|
@@ -309,6 +316,25 @@ function isSafeHtmlUrl(href, image = false) {
|
|
|
309
316
|
}
|
|
310
317
|
return true;
|
|
311
318
|
}
|
|
319
|
+
function escapeHtml(value, attribute = false, protectMarkdown = false) {
|
|
320
|
+
let escaped = "";
|
|
321
|
+
let copiedUntil = 0;
|
|
322
|
+
for (let index = 0; index < value.length; index++) {
|
|
323
|
+
const code = value.charCodeAt(index);
|
|
324
|
+
let replacement;
|
|
325
|
+
if (code === 38) replacement = "&";
|
|
326
|
+
else if (code === 60) replacement = "<";
|
|
327
|
+
else if (code === 62) replacement = ">";
|
|
328
|
+
else if (attribute && code === 34) replacement = """;
|
|
329
|
+
else if (protectMarkdown && code === 91) replacement = "[";
|
|
330
|
+
else if (protectMarkdown && code === 93) replacement = "]";
|
|
331
|
+
if (replacement !== void 0) {
|
|
332
|
+
escaped += value.slice(copiedUntil, index) + replacement;
|
|
333
|
+
copiedUntil = index + 1;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
return copiedUntil === 0 ? value : escaped + value.slice(copiedUntil);
|
|
337
|
+
}
|
|
312
338
|
function continuationPrefix(node, listIndentWidths, includeBlockquotes = true) {
|
|
313
339
|
const chain = [];
|
|
314
340
|
let current = node.parent;
|
|
@@ -384,18 +410,18 @@ function parseUnsignedInteger(raw) {
|
|
|
384
410
|
const value = Number(raw);
|
|
385
411
|
return value <= 4294967295 ? value : void 0;
|
|
386
412
|
}
|
|
387
|
-
function lastOutputChar(buffer) {
|
|
388
|
-
for (let index =
|
|
413
|
+
function lastOutputChar(buffer, end = buffer.length) {
|
|
414
|
+
for (let index = end - 1; index >= 0; index--) {
|
|
389
415
|
const fragment = buffer[index];
|
|
390
416
|
if (fragment.length > 0) return fragment.charCodeAt(fragment.length - 1);
|
|
391
417
|
}
|
|
392
418
|
return -1;
|
|
393
419
|
}
|
|
394
|
-
function blockOpenPrefix(buffer, prefix) {
|
|
395
|
-
switch (lastOutputChar(buffer)) {
|
|
420
|
+
function blockOpenPrefix(buffer, prefix, end = buffer.length) {
|
|
421
|
+
switch (lastOutputChar(buffer, end)) {
|
|
396
422
|
case -1: return;
|
|
397
423
|
case 32: {
|
|
398
|
-
let fragment =
|
|
424
|
+
let fragment = end - 1;
|
|
399
425
|
let index = buffer[fragment].length;
|
|
400
426
|
for (;;) {
|
|
401
427
|
if (index === 0) {
|
|
@@ -411,6 +437,15 @@ function blockOpenPrefix(buffer, prefix) {
|
|
|
411
437
|
default: return `\n\n${prefix ?? ""}`;
|
|
412
438
|
}
|
|
413
439
|
}
|
|
440
|
+
function figcaptionOwnsBlockSpacing(node, plainText) {
|
|
441
|
+
let parent = node.parent;
|
|
442
|
+
while (parent) {
|
|
443
|
+
if (parent.tagId === 32 || parent.tagId === 31) return false;
|
|
444
|
+
if (!plainText && parent.tagId !== 37 && parent.tagHandler?.collapsesInnerWhiteSpace) return false;
|
|
445
|
+
parent = parent.parent;
|
|
446
|
+
}
|
|
447
|
+
return true;
|
|
448
|
+
}
|
|
414
449
|
function listMarkerLineStart(buffer, fragmentIndex, markerIndex) {
|
|
415
450
|
const marker = buffer[fragmentIndex].charCodeAt(markerIndex);
|
|
416
451
|
const ordered = marker === 46 || marker === 41;
|
|
@@ -523,7 +558,8 @@ function decodeHTMLEntities(text, inAttribute = false, protectDecodedEntityRefer
|
|
|
523
558
|
let legacyEnd = Math.min(nameEnd, nameStart + 6);
|
|
524
559
|
let decodedLegacy = false;
|
|
525
560
|
while (legacyEnd > nameStart) {
|
|
526
|
-
const
|
|
561
|
+
const name = text.slice(nameStart, legacyEnd);
|
|
562
|
+
const replacement = HTML_ENTITIES[name];
|
|
527
563
|
if (typeof replacement === "string") {
|
|
528
564
|
const next = text.charCodeAt(legacyEnd);
|
|
529
565
|
if (replacement !== void 0 && !(inAttribute && (next === 61 || isAsciiAlphaNumeric(next)))) {
|
|
@@ -541,16 +577,41 @@ function decodeHTMLEntities(text, inAttribute = false, protectDecodedEntityRefer
|
|
|
541
577
|
}
|
|
542
578
|
return result;
|
|
543
579
|
}
|
|
580
|
+
function isInlineNode(node) {
|
|
581
|
+
const isInline = node.tagHandler?.isInline;
|
|
582
|
+
return isInline === void 0 ? node.tagId === -1 : isInline;
|
|
583
|
+
}
|
|
544
584
|
function traverseUpToFirstBlockNode(node) {
|
|
545
585
|
let firstBlockParent = node;
|
|
546
586
|
const parentsToIncrement = [firstBlockParent];
|
|
547
|
-
while (firstBlockParent
|
|
587
|
+
while (isInlineNode(firstBlockParent)) {
|
|
548
588
|
if (!firstBlockParent.parent) break;
|
|
549
589
|
firstBlockParent = firstBlockParent.parent;
|
|
550
590
|
parentsToIncrement.push(firstBlockParent);
|
|
551
591
|
}
|
|
552
592
|
return parentsToIncrement;
|
|
553
593
|
}
|
|
594
|
+
function markRenderedChildContent(node) {
|
|
595
|
+
let parent = node.parent;
|
|
596
|
+
let anchorInScope = false;
|
|
597
|
+
while (parent) {
|
|
598
|
+
parent.childTextNodeIndex = (parent.childTextNodeIndex || 0) + 1;
|
|
599
|
+
if (parent.tagId === 26) anchorInScope = true;
|
|
600
|
+
if (!isInlineNode(parent)) break;
|
|
601
|
+
parent = parent.parent;
|
|
602
|
+
}
|
|
603
|
+
if (anchorInScope) return;
|
|
604
|
+
let anchorFound = false;
|
|
605
|
+
while (parent?.parent) {
|
|
606
|
+
parent = parent.parent;
|
|
607
|
+
if (!anchorFound) {
|
|
608
|
+
if (parent.tagId !== 26) continue;
|
|
609
|
+
anchorFound = true;
|
|
610
|
+
}
|
|
611
|
+
parent.childTextNodeIndex = (parent.childTextNodeIndex || 0) + 1;
|
|
612
|
+
if (!isInlineNode(parent)) return;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
554
615
|
const TRACKING_PARAM_RE = /^(?:utm_|fbclid|gclid|mc_eid|msclkid|oly_)/;
|
|
555
616
|
const URL_SCHEME_RE = /^[A-Z][\dA-Z+.-]*:/i;
|
|
556
617
|
const SLASH_CHAR$1 = 47;
|
|
@@ -631,6 +692,14 @@ function resolveUrl(url, origin, clean) {
|
|
|
631
692
|
} else if (origin && (isSlash || !URL_SCHEME_RE.test(url))) resolved = resolveAgainstBase(origin, url);
|
|
632
693
|
return (clean === true || !!clean && clean.urls === true) && resolved.includes("?") ? stripTrackingParams(resolved) : resolved;
|
|
633
694
|
}
|
|
695
|
+
function safeAnchorOutput(node, options, entering, protectMarkdown = false) {
|
|
696
|
+
const href = node.attributes?.href;
|
|
697
|
+
if (!href || !isSafeHtmlUrl(href)) return;
|
|
698
|
+
const resolved = resolveUrl(href, options?.origin, options?.clean);
|
|
699
|
+
if (!isSafeHtmlUrl(resolved)) return;
|
|
700
|
+
if (!entering) return "</a>";
|
|
701
|
+
return `<a href="${escapeHtml(resolved, true, protectMarkdown)}"${node.attributes?.title === void 0 ? "" : ` title="${escapeHtml(node.attributes.title, true, protectMarkdown)}"`}>`;
|
|
702
|
+
}
|
|
634
703
|
function serializeMarkdownDestination(destination) {
|
|
635
704
|
if (!/[\t\n\f\r ()\\<>]/.test(destination)) return destination;
|
|
636
705
|
return `<${/[\\<>]/.test(destination) ? destination.replace(/[\\<>]/g, "\\$&") : destination}>`;
|
|
@@ -808,6 +877,7 @@ const tagHandlers = {
|
|
|
808
877
|
exit: ({ state }) => isInsideTableCell(state) ? "</summary>" : "</summary>\n\n"
|
|
809
878
|
},
|
|
810
879
|
[4]: {
|
|
880
|
+
excludesTextNodes: true,
|
|
811
881
|
collapsesInnerWhiteSpace: true,
|
|
812
882
|
isNonNesting: true,
|
|
813
883
|
spacing: NO_SPACING
|
|
@@ -962,12 +1032,14 @@ const tagHandlers = {
|
|
|
962
1032
|
enter: ({ node, state }) => {
|
|
963
1033
|
if (node.attributes?.href !== void 0) {
|
|
964
1034
|
if (stripsEmptyLink(state, node.attributes.href)) return;
|
|
1035
|
+
if (isInsideRawHtmlBlock(state.depthMap)) return safeAnchorOutput(node, state.options, true, true);
|
|
965
1036
|
return "[";
|
|
966
1037
|
}
|
|
967
1038
|
},
|
|
968
1039
|
exit: ({ node, state }) => {
|
|
969
1040
|
if (node.attributes?.href === void 0) return "";
|
|
970
1041
|
if (stripsEmptyLink(state, node.attributes.href)) return "";
|
|
1042
|
+
if (isInsideRawHtmlBlock(state.depthMap) && !node.tagHandler?.literalEnter) return safeAnchorOutput(node, state.options, false, true);
|
|
971
1043
|
const href = resolveUrl(node.attributes.href, state.options?.origin, state.options?.clean);
|
|
972
1044
|
let title = node.attributes?.title;
|
|
973
1045
|
if (state.lastContentCache === title) title = "";
|
|
@@ -975,14 +1047,26 @@ const tagHandlers = {
|
|
|
975
1047
|
const buf = state.buffer;
|
|
976
1048
|
let i = buf.length - 1;
|
|
977
1049
|
let textLen = 0;
|
|
1050
|
+
let captionBracket = -1;
|
|
1051
|
+
let captionTextLen = 0;
|
|
978
1052
|
while (i >= 0) {
|
|
979
1053
|
const entry = buf[i];
|
|
980
1054
|
if (entry === "[") break;
|
|
1055
|
+
if (captionBracket < 0 && state.depthMap?.[106] && entry.endsWith("*[")) {
|
|
1056
|
+
captionBracket = i;
|
|
1057
|
+
captionTextLen = textLen;
|
|
1058
|
+
}
|
|
981
1059
|
textLen += entry.length;
|
|
982
1060
|
i--;
|
|
983
1061
|
}
|
|
1062
|
+
const captionLink = i < 0 && captionBracket >= 0;
|
|
1063
|
+
if (captionLink) {
|
|
1064
|
+
i = captionBracket;
|
|
1065
|
+
textLen = captionTextLen;
|
|
1066
|
+
}
|
|
984
1067
|
if (i >= 0 && textLen === href.length && buf.slice(i + 1).join("") === href) {
|
|
985
|
-
buf
|
|
1068
|
+
if (captionLink) buf[i] = buf[i].slice(0, -1);
|
|
1069
|
+
buf.length = i + (captionLink ? 1 : 0);
|
|
986
1070
|
const auto = `<${href}>`;
|
|
987
1071
|
buf.push(auto);
|
|
988
1072
|
state.lastContentCache = auto;
|
|
@@ -998,7 +1082,12 @@ const tagHandlers = {
|
|
|
998
1082
|
[27]: {
|
|
999
1083
|
enter: ({ node, state }) => {
|
|
1000
1084
|
const alt = node.attributes?.alt || "";
|
|
1001
|
-
const
|
|
1085
|
+
const rawSrc = node.attributes?.src || "";
|
|
1086
|
+
const src = isDataUrl(rawSrc) ? "" : resolveUrl(rawSrc, state.options?.origin, state.options?.clean);
|
|
1087
|
+
const clean = state.options?.clean;
|
|
1088
|
+
if ((clean === true || clean !== void 0 && clean !== false && clean.emptyImages === true) && !alt.trim()) {
|
|
1089
|
+
if (state.depthMap?.[106]) return void 0;
|
|
1090
|
+
} else markRenderedChildContent(node);
|
|
1002
1091
|
return `![${serializeImageDescription(alt)}]${serializeMarkdownResource(src, node.attributes?.title)}`;
|
|
1003
1092
|
},
|
|
1004
1093
|
collapsesInnerWhiteSpace: true,
|
|
@@ -1358,10 +1447,8 @@ const tagHandlers = {
|
|
|
1358
1447
|
[104]: {},
|
|
1359
1448
|
[102]: {},
|
|
1360
1449
|
[106]: {
|
|
1361
|
-
enter: () => "*",
|
|
1362
1450
|
exit: () => "*",
|
|
1363
1451
|
collapsesInnerWhiteSpace: true,
|
|
1364
|
-
spacing: NO_SPACING,
|
|
1365
1452
|
isInline: true
|
|
1366
1453
|
}
|
|
1367
1454
|
};
|
|
@@ -1402,6 +1489,10 @@ function buildTagOverrideHandlers(overrides) {
|
|
|
1402
1489
|
}
|
|
1403
1490
|
return result;
|
|
1404
1491
|
}
|
|
1492
|
+
const UNKNOWN_TAG_HANDLER = {
|
|
1493
|
+
isInline: true,
|
|
1494
|
+
spacing: NO_SPACING
|
|
1495
|
+
};
|
|
1405
1496
|
const LT_CHAR = 60;
|
|
1406
1497
|
const GT_CHAR = 62;
|
|
1407
1498
|
const SLASH_CHAR = 47;
|
|
@@ -1931,24 +2022,26 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1931
2022
|
break;
|
|
1932
2023
|
}
|
|
1933
2024
|
const nextCharCode = htmlChunk.charCodeAt(i + 1);
|
|
1934
|
-
if (state.flattenedRawTagName)
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
2025
|
+
if (state.flattenedRawTagName) {
|
|
2026
|
+
if (nextCharCode === SLASH_CHAR) {
|
|
2027
|
+
let peekEnd = i + 2;
|
|
2028
|
+
while (peekEnd < chunkLength) {
|
|
2029
|
+
const c = htmlChunk.charCodeAt(peekEnd);
|
|
2030
|
+
if (c === GT_CHAR || c === SLASH_CHAR || isWhitespace(c)) break;
|
|
2031
|
+
peekEnd++;
|
|
2032
|
+
}
|
|
2033
|
+
if (peekEnd === chunkLength) {
|
|
2034
|
+
textBuffer += htmlChunk.substring(i);
|
|
2035
|
+
break;
|
|
2036
|
+
}
|
|
2037
|
+
if (normalizeTagName(htmlChunk.substring(i + 2, peekEnd)) !== state.flattenedRawTagName) {
|
|
2038
|
+
textBuffer += htmlChunk[i++];
|
|
2039
|
+
continue;
|
|
2040
|
+
}
|
|
2041
|
+
} else {
|
|
1946
2042
|
textBuffer += htmlChunk[i++];
|
|
1947
2043
|
continue;
|
|
1948
2044
|
}
|
|
1949
|
-
} else {
|
|
1950
|
-
textBuffer += htmlChunk[i++];
|
|
1951
|
-
continue;
|
|
1952
2045
|
}
|
|
1953
2046
|
if (nextCharCode === EXCLAMATION_CHAR) {
|
|
1954
2047
|
if (htmlChunk.charCodeAt(i + 2) === OPEN_BRACKET_CHAR) {
|
|
@@ -2427,7 +2520,7 @@ function processCdataSection(content, state, handleEvent) {
|
|
|
2427
2520
|
function processOpeningTag(tagName, tagId, htmlChunk, i, state, handleEvent) {
|
|
2428
2521
|
tagId = effectiveTagId(tagName, tagId, state);
|
|
2429
2522
|
if (state.currentNode?.tagHandler?.isNonNesting) closeNode(state.currentNode, state, handleEvent);
|
|
2430
|
-
const tagHandler = state.tagOverrideHandlers?.get(tagName) ?? tagHandlers[tagId];
|
|
2523
|
+
const tagHandler = state.tagOverrideHandlers?.get(tagName) ?? (tagId === -1 ? UNKNOWN_TAG_HANDLER : tagHandlers[tagId]);
|
|
2431
2524
|
const result = scanTagAttributes(htmlChunk, i, tagHandler);
|
|
2432
2525
|
if (result._tag === "incomplete") return {
|
|
2433
2526
|
complete: false,
|
|
@@ -2454,37 +2547,41 @@ function processOpeningTag(tagName, tagId, htmlChunk, i, state, handleEvent) {
|
|
|
2454
2547
|
const headNode = state.currentNode;
|
|
2455
2548
|
if (headNode && headNode.tagId === 1) closeNode(headNode, state, handleEvent);
|
|
2456
2549
|
}
|
|
2457
|
-
if (tagId >= 0 && tagId < 113 && NEEDS_IMPLIED_END_RECOVERY[tagId] === 1)
|
|
2458
|
-
if (
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
if (
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
if (
|
|
2473
|
-
|
|
2474
|
-
if (
|
|
2475
|
-
if (parserTagDepth(state,
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
if (
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2550
|
+
if (tagId >= 0 && tagId < 113 && NEEDS_IMPLIED_END_RECOVERY[tagId] === 1) {
|
|
2551
|
+
if (tagId === 65 && parserTagDepth(state, 65) > 0) {
|
|
2552
|
+
if (closeSelectTo(state, 65, handleEvent)) return {
|
|
2553
|
+
complete: true,
|
|
2554
|
+
newPosition: result.newPosition,
|
|
2555
|
+
remainingText: "",
|
|
2556
|
+
selfClosing: false,
|
|
2557
|
+
skip: true
|
|
2558
|
+
};
|
|
2559
|
+
} else if (tagId === 67) {
|
|
2560
|
+
if (parserTagDepth(state, 65) > 0) closeSelectTo(state, 67, handleEvent);
|
|
2561
|
+
else if (parserLastTagId(state) === 67) popParserTop(state, handleEvent);
|
|
2562
|
+
} else if (tagId === 109 && parserTagDepth(state, 65) > 0) {
|
|
2563
|
+
closeSelectTo(state, 67, handleEvent);
|
|
2564
|
+
closeSelectTo(state, 109, handleEvent);
|
|
2565
|
+
} else if (tagId === 26) {
|
|
2566
|
+
if (parserTagDepth(state, 26) > 0) closeImpliedTo(state, SINGLE_A, A_SCOPE_BOUNDARY, handleEvent);
|
|
2567
|
+
} else if (tagId === 32 || tagId === 31 || tagId === 30 || tagId === 29 || tagId === 38 || tagId === 39) {
|
|
2568
|
+
if (parserTagDepth(state, 28) > 0) {
|
|
2569
|
+
if (tagId === 32 || tagId === 31) {
|
|
2570
|
+
if (parserTagDepth(state, 32) > 0 || parserTagDepth(state, 31) > 0) closeImpliedTo(state, TD_TH, CELL_SCOPE_BOUNDARY, handleEvent);
|
|
2571
|
+
} else if (tagId === 30) {
|
|
2572
|
+
if (parserTagDepth(state, 30) > 0) closeTableContext(state, TR_CELLS, handleEvent);
|
|
2573
|
+
} else closeTableContext(state, SECTION_CELLS, handleEvent);
|
|
2574
|
+
}
|
|
2575
|
+
} else {
|
|
2576
|
+
if (parserTagDepth(state, 35) > 0) closeImpliedTo(state, SINGLE_P, P_SCOPE_BOUNDARY, handleEvent);
|
|
2577
|
+
if (HEADINGS.has(tagId)) {
|
|
2578
|
+
const topId = parserLastTagId(state);
|
|
2579
|
+
if (topId !== void 0 && HEADINGS.has(topId)) popParserTop(state, handleEvent);
|
|
2580
|
+
} else if (tagId === 25) {
|
|
2581
|
+
if (parserTagDepth(state, 25) > 0) closeImpliedTo(state, SINGLE_LI, LI_SCOPE_BOUNDARY, handleEvent);
|
|
2582
|
+
} else if (tagId === 99 || tagId === 98) {
|
|
2583
|
+
if (parserTagDepth(state, 99) > 0 || parserTagDepth(state, 98) > 0) closeImpliedTo(state, DT_DD, DL_SCOPE_BOUNDARY, handleEvent);
|
|
2584
|
+
}
|
|
2488
2585
|
}
|
|
2489
2586
|
}
|
|
2490
2587
|
if (state.flattenedTagName && !result.selfClosing) {
|
|
@@ -2618,12 +2715,10 @@ function scanAttributes(source, position, selfClosing, stopAtTagEnd) {
|
|
|
2618
2715
|
state = ATTRIBUTE_GAP;
|
|
2619
2716
|
}
|
|
2620
2717
|
break;
|
|
2621
|
-
case ATTRIBUTE_UNQUOTED_VALUE:
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
}
|
|
2626
|
-
break;
|
|
2718
|
+
case ATTRIBUTE_UNQUOTED_VALUE: if (isSpace || !stopAtTagEnd && charCode === GT_CHAR) {
|
|
2719
|
+
attributes = setAttribute(attributes, source.substring(nameStart, nameEnd), source.substring(valueStart, i));
|
|
2720
|
+
state = ATTRIBUTE_GAP;
|
|
2721
|
+
}
|
|
2627
2722
|
}
|
|
2628
2723
|
i++;
|
|
2629
2724
|
}
|
|
@@ -2640,9 +2735,7 @@ function scanAttributes(source, position, selfClosing, stopAtTagEnd) {
|
|
|
2640
2735
|
attributes = setAttribute(attributes, source.substring(nameStart, nameEnd), "");
|
|
2641
2736
|
break;
|
|
2642
2737
|
case ATTRIBUTE_QUOTED_VALUE:
|
|
2643
|
-
case ATTRIBUTE_UNQUOTED_VALUE:
|
|
2644
|
-
attributes = setAttribute(attributes, source.substring(nameStart, nameEnd), source.substring(valueStart, valueEnd));
|
|
2645
|
-
break;
|
|
2738
|
+
case ATTRIBUTE_UNQUOTED_VALUE: attributes = setAttribute(attributes, source.substring(nameStart, nameEnd), source.substring(valueStart, valueEnd));
|
|
2646
2739
|
}
|
|
2647
2740
|
return {
|
|
2648
2741
|
_tag: "complete",
|
|
@@ -2660,4 +2753,4 @@ function parseAttributes(attrStr) {
|
|
|
2660
2753
|
if (result._tag === "incomplete") throw new Error("Isolated attribute scans always complete at EOF");
|
|
2661
2754
|
return result.attributes === EMPTY_ATTRIBUTES ? {} : result.attributes;
|
|
2662
2755
|
}
|
|
2663
|
-
export { blockOpenPrefix, breakHandler, buildTagOverrideHandlers, continuationPrefix, finalizeParse, getLanguageFromClass, isCharacterReferenceTail, isEmptyLinkHref, isInsideHeading, isInsideTableCell, isSafeHtmlUrl, listMarkerLineStart, orderedItemNumber, parseAttributes, parseHtml, parseHtmlStream, parseUnsignedInteger, renderBreak, resolveUrl };
|
|
2756
|
+
export { blockOpenPrefix, breakHandler, buildTagOverrideHandlers, continuationPrefix, escapeHtml, figcaptionOwnsBlockSpacing, finalizeParse, getLanguageFromClass, isCharacterReferenceTail, isDataUrl, isEmptyLinkHref, isInsideHeading, isInsideTableCell, isSafeHtmlUrl, lastOutputChar, listMarkerLineStart, markRenderedChildContent, orderedItemNumber, parseAttributes, parseHtml, parseHtmlStream, parseUnsignedInteger, renderBreak, resolveUrl, safeAnchorOutput };
|
package/dist/_chunks/stream.mjs
CHANGED
|
@@ -19,13 +19,15 @@ async function* streamHtmlToMarkdown(htmlStream, options = {}, resolvedPlugins =
|
|
|
19
19
|
const { done, value } = await reader.read();
|
|
20
20
|
if (done) break;
|
|
21
21
|
const decoded = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
|
|
22
|
-
|
|
22
|
+
const htmlContent = `${remainingHtml}${decoded}`;
|
|
23
|
+
remainingHtml = parseHtmlStream(htmlContent, parseState, handleEvent);
|
|
23
24
|
const chunk = processor.getMarkdownChunk();
|
|
24
25
|
if (chunk) yield chunk;
|
|
25
26
|
}
|
|
26
27
|
const decoderTail = decoder.decode();
|
|
27
28
|
const finalHtml = remainingHtml + decoderTail;
|
|
28
|
-
|
|
29
|
+
const leftover = finalHtml ? parseHtmlStream(finalHtml, parseState, handleEvent) : "";
|
|
30
|
+
finalizeParse(leftover, parseState, handleEvent);
|
|
29
31
|
const finalChunk = processor.getMarkdownChunk();
|
|
30
32
|
if (finalChunk) yield finalChunk;
|
|
31
33
|
} finally {
|
package/dist/_chunks/types.d.mts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
declare const TAG_H1 = 7;
|
|
2
|
-
declare const TAG_H2 = 8;
|
|
3
|
-
declare const TAG_H3 = 9;
|
|
4
|
-
declare const TAG_H4 = 10;
|
|
5
|
-
declare const TAG_H5 = 11;
|
|
6
|
-
declare const TAG_H6 = 12;
|
|
7
|
-
declare const ELEMENT_NODE = 1;
|
|
8
|
-
declare const TEXT_NODE = 2;
|
|
1
|
+
export declare const TAG_H1 = 7;
|
|
2
|
+
export declare const TAG_H2 = 8;
|
|
3
|
+
export declare const TAG_H3 = 9;
|
|
4
|
+
export declare const TAG_H4 = 10;
|
|
5
|
+
export declare const TAG_H5 = 11;
|
|
6
|
+
export declare const TAG_H6 = 12;
|
|
7
|
+
export declare const ELEMENT_NODE = 1;
|
|
8
|
+
export declare const TEXT_NODE = 2;
|
|
9
9
|
declare const NodeEventEnter$1 = 0;
|
|
10
10
|
declare const NodeEventExit$1 = 1;
|
|
11
11
|
/**
|
|
@@ -13,7 +13,7 @@ declare const NodeEventExit$1 = 1;
|
|
|
13
13
|
* When transforms are provided, the JS engine is used regardless of engine selection.
|
|
14
14
|
* For declarative config that works with both engines, use `BuiltinPlugins`.
|
|
15
15
|
*/
|
|
16
|
-
interface TransformPlugin {
|
|
16
|
+
export interface TransformPlugin {
|
|
17
17
|
/**
|
|
18
18
|
* Whether a flattened element starts an output-excluded subtree.
|
|
19
19
|
* @internal
|
|
@@ -51,7 +51,7 @@ interface TransformPlugin {
|
|
|
51
51
|
* Declarative tag override configuration.
|
|
52
52
|
* When a string value is provided, it acts as an alias (e.g. `{ "x-heading": "h2" }`).
|
|
53
53
|
*/
|
|
54
|
-
interface TagOverride {
|
|
54
|
+
export interface TagOverride {
|
|
55
55
|
enter?: string;
|
|
56
56
|
exit?: string;
|
|
57
57
|
spacing?: [number, number];
|
|
@@ -62,7 +62,7 @@ interface TagOverride {
|
|
|
62
62
|
/**
|
|
63
63
|
* Frontmatter configuration options.
|
|
64
64
|
*/
|
|
65
|
-
interface FrontmatterConfig {
|
|
65
|
+
export interface FrontmatterConfig {
|
|
66
66
|
additionalFields?: Record<string, string>;
|
|
67
67
|
metaFields?: string[];
|
|
68
68
|
/**
|
|
@@ -75,9 +75,13 @@ interface FrontmatterConfig {
|
|
|
75
75
|
* Declarative configuration for built-in plugins.
|
|
76
76
|
* Works with both the JavaScript and Rust engines.
|
|
77
77
|
*/
|
|
78
|
-
interface BuiltinPlugins {
|
|
79
|
-
/**
|
|
80
|
-
|
|
78
|
+
export interface BuiltinPlugins {
|
|
79
|
+
/**
|
|
80
|
+
* Filter elements by CSS selectors, tag names, or TAG_* constants. With
|
|
81
|
+
* `withMinimalPreset()`, a filter adds to the preset's excludes; `false`
|
|
82
|
+
* turns filtering off.
|
|
83
|
+
*/
|
|
84
|
+
filter?: false | {
|
|
81
85
|
include?: (string | number)[];
|
|
82
86
|
exclude?: (string | number)[];
|
|
83
87
|
processChildren?: boolean;
|
|
@@ -109,7 +113,7 @@ interface BuiltinPlugins {
|
|
|
109
113
|
* Shared engine options that work with both JS and Rust engines.
|
|
110
114
|
* This is the contract that `MarkdownEngine` methods accept.
|
|
111
115
|
*/
|
|
112
|
-
interface CleanOptions {
|
|
116
|
+
export interface CleanOptions {
|
|
113
117
|
/** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
|
|
114
118
|
urls?: boolean;
|
|
115
119
|
/** Strip fragment-only links that don't match any heading in the output */
|
|
@@ -127,7 +131,7 @@ interface CleanOptions {
|
|
|
127
131
|
/** Drop links that produce no visible text: [](url) → nothing */
|
|
128
132
|
emptyLinkText?: boolean;
|
|
129
133
|
}
|
|
130
|
-
interface EngineOptions {
|
|
134
|
+
export interface EngineOptions {
|
|
131
135
|
/**
|
|
132
136
|
* Origin URL for resolving relative image paths and internal links.
|
|
133
137
|
*/
|
|
@@ -155,7 +159,7 @@ interface EngineOptions {
|
|
|
155
159
|
*/
|
|
156
160
|
format?: 'markdown' | 'text' | 'html';
|
|
157
161
|
}
|
|
158
|
-
interface ElementNode extends Node {
|
|
162
|
+
export interface ElementNode extends Node {
|
|
159
163
|
/** Element tag name (for ELEMENT_NODE) */
|
|
160
164
|
name: string;
|
|
161
165
|
/** HTML attributes (for ELEMENT_NODE) */
|
|
@@ -169,7 +173,7 @@ interface ElementNode extends Node {
|
|
|
169
173
|
/** Plugin outputs collected during processing */
|
|
170
174
|
pluginOutput?: string[];
|
|
171
175
|
}
|
|
172
|
-
interface TextNode extends Node {
|
|
176
|
+
export interface TextNode extends Node {
|
|
173
177
|
/** Text content (for TEXT_NODE) */
|
|
174
178
|
value: string;
|
|
175
179
|
/** Markdown delimiters generated by a plugin around this literal text. */
|
|
@@ -184,7 +188,7 @@ interface TextNode extends Node {
|
|
|
184
188
|
* Base DOM node interface
|
|
185
189
|
* Optimized for streaming HTML parsing with minimal memory footprint
|
|
186
190
|
*/
|
|
187
|
-
interface Node {
|
|
191
|
+
export interface Node {
|
|
188
192
|
/** Node type (ELEMENT_NODE or TEXT_NODE) */
|
|
189
193
|
type: number;
|
|
190
194
|
/** Current nesting depth in the DOM tree */
|
|
@@ -194,7 +198,7 @@ interface Node {
|
|
|
194
198
|
index: number;
|
|
195
199
|
/** Current walk index for child traversal during streaming */
|
|
196
200
|
currentWalkIndex?: number;
|
|
197
|
-
/**
|
|
201
|
+
/** Child-content counter used by whitespace and empty-link handling */
|
|
198
202
|
childTextNodeIndex?: number;
|
|
199
203
|
/** Whether node contains whitespace - used for whitespace optimization */
|
|
200
204
|
containsWhitespace?: boolean;
|
|
@@ -250,7 +254,7 @@ interface MdreamProcessingState {
|
|
|
250
254
|
* Runtime state for markdown generation
|
|
251
255
|
* Extended state that includes output tracking and options
|
|
252
256
|
*/
|
|
253
|
-
interface MdreamRuntimeState extends Partial<MdreamProcessingState> {
|
|
257
|
+
export interface MdreamRuntimeState extends Partial<MdreamProcessingState> {
|
|
254
258
|
/** Number of newlines at end of most recent output */
|
|
255
259
|
lastNewLines?: number;
|
|
256
260
|
/** Configuration options for conversion */
|
|
@@ -296,7 +300,7 @@ type NodeEventExit = 1;
|
|
|
296
300
|
* Node event for DOM traversal
|
|
297
301
|
* Used in the event-based traversal system for streaming processing
|
|
298
302
|
*/
|
|
299
|
-
interface NodeEvent {
|
|
303
|
+
export interface NodeEvent {
|
|
300
304
|
/** Event type - enter (start tag) or exit (end tag) */
|
|
301
305
|
type: NodeEventEnter | NodeEventExit;
|
|
302
306
|
/** The node being processed */
|
|
@@ -340,7 +344,7 @@ type TagHandlerResult = string | GfmAction | undefined | void;
|
|
|
340
344
|
* Tag handler interface for HTML elements
|
|
341
345
|
* Used by plugins to extend or customize tag handling
|
|
342
346
|
*/
|
|
343
|
-
interface TagHandler {
|
|
347
|
+
export interface TagHandler {
|
|
344
348
|
enter?: (context: HandlerContext) => TagHandlerResult;
|
|
345
349
|
exit?: (context: HandlerContext) => TagHandlerResult;
|
|
346
350
|
isSelfClosing?: boolean;
|
|
@@ -368,7 +372,7 @@ interface TailwindContext {
|
|
|
368
372
|
prefix?: string;
|
|
369
373
|
suffix?: string;
|
|
370
374
|
}
|
|
371
|
-
interface PluginContext {
|
|
375
|
+
export interface PluginContext {
|
|
372
376
|
score?: number;
|
|
373
377
|
tagCount?: number;
|
|
374
378
|
linkTextLength?: number;
|
|
@@ -380,7 +384,7 @@ interface PluginContext {
|
|
|
380
384
|
/**
|
|
381
385
|
* Element extracted during conversion by the extraction plugin.
|
|
382
386
|
*/
|
|
383
|
-
interface ExtractedElement {
|
|
387
|
+
export interface ExtractedElement {
|
|
384
388
|
/** The CSS selector that matched this element */
|
|
385
389
|
selector: string;
|
|
386
390
|
/** The HTML tag name */
|
|
@@ -394,7 +398,7 @@ interface ExtractedElement {
|
|
|
394
398
|
* Top-level options for the mdream JS engine.
|
|
395
399
|
* Extends the shared `EngineOptions` with JS-specific concerns.
|
|
396
400
|
*/
|
|
397
|
-
interface MdreamOptions extends EngineOptions {
|
|
401
|
+
export interface MdreamOptions extends EngineOptions {
|
|
398
402
|
/**
|
|
399
403
|
* Imperative hook-based transform plugins.
|
|
400
404
|
* When provided, a new JS engine is created with these hooks.
|
|
@@ -405,7 +409,7 @@ interface MdreamOptions extends EngineOptions {
|
|
|
405
409
|
* Markdown chunk with content and metadata
|
|
406
410
|
* Compatible with LangChain Document structure
|
|
407
411
|
*/
|
|
408
|
-
interface MarkdownChunk {
|
|
412
|
+
export interface MarkdownChunk {
|
|
409
413
|
/** The markdown content of the chunk */
|
|
410
414
|
content: string;
|
|
411
415
|
/** Metadata extracted during chunking */
|
|
@@ -427,7 +431,7 @@ interface MarkdownChunk {
|
|
|
427
431
|
* Options for HTML to Markdown chunking
|
|
428
432
|
* Extends EngineOptions with chunking-specific settings
|
|
429
433
|
*/
|
|
430
|
-
interface SplitterOptions extends EngineOptions {
|
|
434
|
+
export interface SplitterOptions extends EngineOptions {
|
|
431
435
|
/**
|
|
432
436
|
* Header tag IDs to split on (TAG_H1, TAG_H2, etc.)
|
|
433
437
|
* @example [TAG_H1, TAG_H2]
|
|
@@ -466,4 +470,4 @@ interface SplitterOptions extends EngineOptions {
|
|
|
466
470
|
*/
|
|
467
471
|
keepSeparator?: boolean;
|
|
468
472
|
}
|
|
469
|
-
export {
|
|
473
|
+
export { NodeEventEnter$1 as NodeEventEnter, NodeEventExit$1 as NodeEventExit };
|
package/dist/cli.d.mts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export {}
|
package/dist/cli.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { streamHtmlToMarkdown } from "./_chunks/src.mjs";
|
|
2
1
|
import { withMinimalPreset } from "./preset/minimal.mjs";
|
|
2
|
+
import { streamHtmlToMarkdown } from "./index.mjs";
|
|
3
3
|
import { readFileSync } from "node:fs";
|
|
4
4
|
import { Readable } from "node:stream";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
@@ -21,7 +21,8 @@ async function streamingConvert(options = {}) {
|
|
|
21
21
|
const markdownGenerator = streamHtmlToMarkdown(Readable.toWeb(process.stdin), conversionOptions);
|
|
22
22
|
for await (const markdownChunk of markdownGenerator) if (markdownChunk && markdownChunk.length > 0) process.stdout.write(markdownChunk);
|
|
23
23
|
}
|
|
24
|
-
const
|
|
24
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
25
|
+
const packageJsonPath = join(__dirname, "..", "package.json");
|
|
25
26
|
const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
|
|
26
27
|
const cli = cac();
|
|
27
28
|
cli.command("[options]", "Convert HTML from stdin to Markdown on stdout (JS engine)").option("--origin <url>", "Origin URL for resolving relative image paths").option("--preset <preset>", "Conversion presets: minimal").option("--wrap-width <n>", "Hard-wrap prose at <n> characters on word boundaries").option("--format <format>", "Output format: markdown, text, html").option("--text", "Alias for --format text").action(async (_, opts) => {
|
package/dist/core.d.mts
CHANGED
|
@@ -1,16 +1,15 @@
|
|
|
1
1
|
import { BuiltinPlugins, MdreamOptions, TransformPlugin } from "./_chunks/types.mjs";
|
|
2
2
|
/** Imperative plugin used by the tree-shakable converter. */
|
|
3
|
-
type Plugin = TransformPlugin;
|
|
3
|
+
export type Plugin = TransformPlugin;
|
|
4
4
|
/**
|
|
5
5
|
* Options for the tree-shakable conversion entry. Import plugin factories
|
|
6
6
|
* explicitly and pass their results as an array, matching the pre-v1 JS API.
|
|
7
7
|
* Declarative built-in plugin config and cleanup live on the package root.
|
|
8
8
|
*/
|
|
9
|
-
type CoreOptions = Omit<MdreamOptions, 'plugins' | 'clean'> & {
|
|
9
|
+
export type CoreOptions = Omit<MdreamOptions, 'plugins' | 'clean'> & {
|
|
10
10
|
plugins?: Plugin[] | Pick<BuiltinPlugins, 'tagOverrides'>;
|
|
11
11
|
};
|
|
12
12
|
/** Convert HTML without bundling the optional declarative built-in plugins. */
|
|
13
|
-
declare function htmlToMarkdown(html: string, options?: CoreOptions): string;
|
|
13
|
+
export declare function htmlToMarkdown(html: string, options?: CoreOptions): string;
|
|
14
14
|
/** Stream HTML conversion without bundling declarative built-in plugins. */
|
|
15
|
-
declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: CoreOptions): AsyncIterable<string>;
|
|
16
|
-
export { CoreOptions, Plugin, htmlToMarkdown, streamHtmlToMarkdown };
|
|
15
|
+
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: CoreOptions): AsyncIterable<string>;
|