@mdream/js 1.7.1 → 1.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/index.d.mts +3 -4
- package/dist/_chunks/markdown-processor.mjs +905 -131
- package/dist/_chunks/minimal.d.mts +1 -2
- package/dist/_chunks/parse.mjs +146 -66
- package/dist/_chunks/stream.mjs +4 -2
- package/dist/_chunks/types.d.mts +27 -27
- package/dist/cli.d.mts +1 -1
- package/dist/cli.mjs +3 -2
- package/dist/core.d.mts +4 -5
- package/dist/core.mjs +4 -2
- package/dist/index.mjs +30 -1
- package/dist/llms-txt.d.mts +8 -9
- package/dist/negotiate.d.mts +4 -5
- package/dist/parse.d.mts +7 -8
- package/dist/plugins.d.mts +7 -7
- package/dist/splitter.d.mts +3 -3
- package/package.json +1 -1
- package/dist/_chunks/src.mjs +0 -316
|
@@ -3,5 +3,4 @@ import { EngineOptions } from "./types.mjs";
|
|
|
3
3
|
* Creates a configurable minimal preset with advanced options.
|
|
4
4
|
* Returns declarative plugin config that works with both JS and Rust engines.
|
|
5
5
|
*/
|
|
6
|
-
declare function withMinimalPreset<T extends EngineOptions>(options?: T): T;
|
|
7
|
-
export { withMinimalPreset };
|
|
6
|
+
export declare function withMinimalPreset<T extends EngineOptions>(options?: T): T;
|
package/dist/_chunks/parse.mjs
CHANGED
|
@@ -309,6 +309,25 @@ function isSafeHtmlUrl(href, image = false) {
|
|
|
309
309
|
}
|
|
310
310
|
return true;
|
|
311
311
|
}
|
|
312
|
+
function escapeHtml(value, attribute = false, protectMarkdown = false) {
|
|
313
|
+
let escaped = "";
|
|
314
|
+
let copiedUntil = 0;
|
|
315
|
+
for (let index = 0; index < value.length; index++) {
|
|
316
|
+
const code = value.charCodeAt(index);
|
|
317
|
+
let replacement;
|
|
318
|
+
if (code === 38) replacement = "&";
|
|
319
|
+
else if (code === 60) replacement = "<";
|
|
320
|
+
else if (code === 62) replacement = ">";
|
|
321
|
+
else if (attribute && code === 34) replacement = """;
|
|
322
|
+
else if (protectMarkdown && code === 91) replacement = "[";
|
|
323
|
+
else if (protectMarkdown && code === 93) replacement = "]";
|
|
324
|
+
if (replacement !== void 0) {
|
|
325
|
+
escaped += value.slice(copiedUntil, index) + replacement;
|
|
326
|
+
copiedUntil = index + 1;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return copiedUntil === 0 ? value : escaped + value.slice(copiedUntil);
|
|
330
|
+
}
|
|
312
331
|
function continuationPrefix(node, listIndentWidths, includeBlockquotes = true) {
|
|
313
332
|
const chain = [];
|
|
314
333
|
let current = node.parent;
|
|
@@ -384,18 +403,18 @@ function parseUnsignedInteger(raw) {
|
|
|
384
403
|
const value = Number(raw);
|
|
385
404
|
return value <= 4294967295 ? value : void 0;
|
|
386
405
|
}
|
|
387
|
-
function lastOutputChar(buffer) {
|
|
388
|
-
for (let index =
|
|
406
|
+
function lastOutputChar(buffer, end = buffer.length) {
|
|
407
|
+
for (let index = end - 1; index >= 0; index--) {
|
|
389
408
|
const fragment = buffer[index];
|
|
390
409
|
if (fragment.length > 0) return fragment.charCodeAt(fragment.length - 1);
|
|
391
410
|
}
|
|
392
411
|
return -1;
|
|
393
412
|
}
|
|
394
|
-
function blockOpenPrefix(buffer, prefix) {
|
|
395
|
-
switch (lastOutputChar(buffer)) {
|
|
413
|
+
function blockOpenPrefix(buffer, prefix, end = buffer.length) {
|
|
414
|
+
switch (lastOutputChar(buffer, end)) {
|
|
396
415
|
case -1: return;
|
|
397
416
|
case 32: {
|
|
398
|
-
let fragment =
|
|
417
|
+
let fragment = end - 1;
|
|
399
418
|
let index = buffer[fragment].length;
|
|
400
419
|
for (;;) {
|
|
401
420
|
if (index === 0) {
|
|
@@ -411,6 +430,15 @@ function blockOpenPrefix(buffer, prefix) {
|
|
|
411
430
|
default: return `\n\n${prefix ?? ""}`;
|
|
412
431
|
}
|
|
413
432
|
}
|
|
433
|
+
function figcaptionOwnsBlockSpacing(node, plainText) {
|
|
434
|
+
let parent = node.parent;
|
|
435
|
+
while (parent) {
|
|
436
|
+
if (parent.tagId === 32 || parent.tagId === 31) return false;
|
|
437
|
+
if (!plainText && parent.tagId !== 37 && parent.tagHandler?.collapsesInnerWhiteSpace) return false;
|
|
438
|
+
parent = parent.parent;
|
|
439
|
+
}
|
|
440
|
+
return true;
|
|
441
|
+
}
|
|
414
442
|
function listMarkerLineStart(buffer, fragmentIndex, markerIndex) {
|
|
415
443
|
const marker = buffer[fragmentIndex].charCodeAt(markerIndex);
|
|
416
444
|
const ordered = marker === 46 || marker === 41;
|
|
@@ -523,7 +551,8 @@ function decodeHTMLEntities(text, inAttribute = false, protectDecodedEntityRefer
|
|
|
523
551
|
let legacyEnd = Math.min(nameEnd, nameStart + 6);
|
|
524
552
|
let decodedLegacy = false;
|
|
525
553
|
while (legacyEnd > nameStart) {
|
|
526
|
-
const
|
|
554
|
+
const name = text.slice(nameStart, legacyEnd);
|
|
555
|
+
const replacement = HTML_ENTITIES[name];
|
|
527
556
|
if (typeof replacement === "string") {
|
|
528
557
|
const next = text.charCodeAt(legacyEnd);
|
|
529
558
|
if (replacement !== void 0 && !(inAttribute && (next === 61 || isAsciiAlphaNumeric(next)))) {
|
|
@@ -541,16 +570,41 @@ function decodeHTMLEntities(text, inAttribute = false, protectDecodedEntityRefer
|
|
|
541
570
|
}
|
|
542
571
|
return result;
|
|
543
572
|
}
|
|
573
|
+
function isInlineNode(node) {
|
|
574
|
+
const isInline = node.tagHandler?.isInline;
|
|
575
|
+
return isInline === void 0 ? node.tagId === -1 : isInline;
|
|
576
|
+
}
|
|
544
577
|
function traverseUpToFirstBlockNode(node) {
|
|
545
578
|
let firstBlockParent = node;
|
|
546
579
|
const parentsToIncrement = [firstBlockParent];
|
|
547
|
-
while (firstBlockParent
|
|
580
|
+
while (isInlineNode(firstBlockParent)) {
|
|
548
581
|
if (!firstBlockParent.parent) break;
|
|
549
582
|
firstBlockParent = firstBlockParent.parent;
|
|
550
583
|
parentsToIncrement.push(firstBlockParent);
|
|
551
584
|
}
|
|
552
585
|
return parentsToIncrement;
|
|
553
586
|
}
|
|
587
|
+
function markRenderedChildContent(node) {
|
|
588
|
+
let parent = node.parent;
|
|
589
|
+
let anchorInScope = false;
|
|
590
|
+
while (parent) {
|
|
591
|
+
parent.childTextNodeIndex = (parent.childTextNodeIndex || 0) + 1;
|
|
592
|
+
if (parent.tagId === 26) anchorInScope = true;
|
|
593
|
+
if (!isInlineNode(parent)) break;
|
|
594
|
+
parent = parent.parent;
|
|
595
|
+
}
|
|
596
|
+
if (anchorInScope) return;
|
|
597
|
+
let anchorFound = false;
|
|
598
|
+
while (parent?.parent) {
|
|
599
|
+
parent = parent.parent;
|
|
600
|
+
if (!anchorFound) {
|
|
601
|
+
if (parent.tagId !== 26) continue;
|
|
602
|
+
anchorFound = true;
|
|
603
|
+
}
|
|
604
|
+
parent.childTextNodeIndex = (parent.childTextNodeIndex || 0) + 1;
|
|
605
|
+
if (!isInlineNode(parent)) return;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
554
608
|
const TRACKING_PARAM_RE = /^(?:utm_|fbclid|gclid|mc_eid|msclkid|oly_)/;
|
|
555
609
|
const URL_SCHEME_RE = /^[A-Z][\dA-Z+.-]*:/i;
|
|
556
610
|
const SLASH_CHAR$1 = 47;
|
|
@@ -631,6 +685,14 @@ function resolveUrl(url, origin, clean) {
|
|
|
631
685
|
} else if (origin && (isSlash || !URL_SCHEME_RE.test(url))) resolved = resolveAgainstBase(origin, url);
|
|
632
686
|
return (clean === true || !!clean && clean.urls === true) && resolved.includes("?") ? stripTrackingParams(resolved) : resolved;
|
|
633
687
|
}
|
|
688
|
+
function safeAnchorOutput(node, options, entering, protectMarkdown = false) {
|
|
689
|
+
const href = node.attributes?.href;
|
|
690
|
+
if (!href || !isSafeHtmlUrl(href)) return;
|
|
691
|
+
const resolved = resolveUrl(href, options?.origin, options?.clean);
|
|
692
|
+
if (!isSafeHtmlUrl(resolved)) return;
|
|
693
|
+
if (!entering) return "</a>";
|
|
694
|
+
return `<a href="${escapeHtml(resolved, true, protectMarkdown)}"${node.attributes?.title === void 0 ? "" : ` title="${escapeHtml(node.attributes.title, true, protectMarkdown)}"`}>`;
|
|
695
|
+
}
|
|
634
696
|
function serializeMarkdownDestination(destination) {
|
|
635
697
|
if (!/[\t\n\f\r ()\\<>]/.test(destination)) return destination;
|
|
636
698
|
return `<${/[\\<>]/.test(destination) ? destination.replace(/[\\<>]/g, "\\$&") : destination}>`;
|
|
@@ -962,12 +1024,14 @@ const tagHandlers = {
|
|
|
962
1024
|
enter: ({ node, state }) => {
|
|
963
1025
|
if (node.attributes?.href !== void 0) {
|
|
964
1026
|
if (stripsEmptyLink(state, node.attributes.href)) return;
|
|
1027
|
+
if (isInsideRawHtmlBlock(state.depthMap)) return safeAnchorOutput(node, state.options, true, true);
|
|
965
1028
|
return "[";
|
|
966
1029
|
}
|
|
967
1030
|
},
|
|
968
1031
|
exit: ({ node, state }) => {
|
|
969
1032
|
if (node.attributes?.href === void 0) return "";
|
|
970
1033
|
if (stripsEmptyLink(state, node.attributes.href)) return "";
|
|
1034
|
+
if (isInsideRawHtmlBlock(state.depthMap) && !node.tagHandler?.literalEnter) return safeAnchorOutput(node, state.options, false, true);
|
|
971
1035
|
const href = resolveUrl(node.attributes.href, state.options?.origin, state.options?.clean);
|
|
972
1036
|
let title = node.attributes?.title;
|
|
973
1037
|
if (state.lastContentCache === title) title = "";
|
|
@@ -975,14 +1039,26 @@ const tagHandlers = {
|
|
|
975
1039
|
const buf = state.buffer;
|
|
976
1040
|
let i = buf.length - 1;
|
|
977
1041
|
let textLen = 0;
|
|
1042
|
+
let captionBracket = -1;
|
|
1043
|
+
let captionTextLen = 0;
|
|
978
1044
|
while (i >= 0) {
|
|
979
1045
|
const entry = buf[i];
|
|
980
1046
|
if (entry === "[") break;
|
|
1047
|
+
if (captionBracket < 0 && state.depthMap?.[106] && entry.endsWith("*[")) {
|
|
1048
|
+
captionBracket = i;
|
|
1049
|
+
captionTextLen = textLen;
|
|
1050
|
+
}
|
|
981
1051
|
textLen += entry.length;
|
|
982
1052
|
i--;
|
|
983
1053
|
}
|
|
1054
|
+
const captionLink = i < 0 && captionBracket >= 0;
|
|
1055
|
+
if (captionLink) {
|
|
1056
|
+
i = captionBracket;
|
|
1057
|
+
textLen = captionTextLen;
|
|
1058
|
+
}
|
|
984
1059
|
if (i >= 0 && textLen === href.length && buf.slice(i + 1).join("") === href) {
|
|
985
|
-
buf
|
|
1060
|
+
if (captionLink) buf[i] = buf[i].slice(0, -1);
|
|
1061
|
+
buf.length = i + (captionLink ? 1 : 0);
|
|
986
1062
|
const auto = `<${href}>`;
|
|
987
1063
|
buf.push(auto);
|
|
988
1064
|
state.lastContentCache = auto;
|
|
@@ -999,6 +1075,10 @@ const tagHandlers = {
|
|
|
999
1075
|
enter: ({ node, state }) => {
|
|
1000
1076
|
const alt = node.attributes?.alt || "";
|
|
1001
1077
|
const src = resolveUrl(node.attributes?.src || "", state.options?.origin, state.options?.clean);
|
|
1078
|
+
const clean = state.options?.clean;
|
|
1079
|
+
if ((clean === true || clean !== void 0 && clean !== false && clean.emptyImages === true) && !alt.trim()) {
|
|
1080
|
+
if (state.depthMap?.[106]) return void 0;
|
|
1081
|
+
} else markRenderedChildContent(node);
|
|
1002
1082
|
return `![${serializeImageDescription(alt)}]${serializeMarkdownResource(src, node.attributes?.title)}`;
|
|
1003
1083
|
},
|
|
1004
1084
|
collapsesInnerWhiteSpace: true,
|
|
@@ -1358,10 +1438,8 @@ const tagHandlers = {
|
|
|
1358
1438
|
[104]: {},
|
|
1359
1439
|
[102]: {},
|
|
1360
1440
|
[106]: {
|
|
1361
|
-
enter: () => "*",
|
|
1362
1441
|
exit: () => "*",
|
|
1363
1442
|
collapsesInnerWhiteSpace: true,
|
|
1364
|
-
spacing: NO_SPACING,
|
|
1365
1443
|
isInline: true
|
|
1366
1444
|
}
|
|
1367
1445
|
};
|
|
@@ -1931,24 +2009,26 @@ function parseHtmlInternal(htmlChunk, state, handleEvent) {
|
|
|
1931
2009
|
break;
|
|
1932
2010
|
}
|
|
1933
2011
|
const nextCharCode = htmlChunk.charCodeAt(i + 1);
|
|
1934
|
-
if (state.flattenedRawTagName)
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
2012
|
+
if (state.flattenedRawTagName) {
|
|
2013
|
+
if (nextCharCode === SLASH_CHAR) {
|
|
2014
|
+
let peekEnd = i + 2;
|
|
2015
|
+
while (peekEnd < chunkLength) {
|
|
2016
|
+
const c = htmlChunk.charCodeAt(peekEnd);
|
|
2017
|
+
if (c === GT_CHAR || c === SLASH_CHAR || isWhitespace(c)) break;
|
|
2018
|
+
peekEnd++;
|
|
2019
|
+
}
|
|
2020
|
+
if (peekEnd === chunkLength) {
|
|
2021
|
+
textBuffer += htmlChunk.substring(i);
|
|
2022
|
+
break;
|
|
2023
|
+
}
|
|
2024
|
+
if (normalizeTagName(htmlChunk.substring(i + 2, peekEnd)) !== state.flattenedRawTagName) {
|
|
2025
|
+
textBuffer += htmlChunk[i++];
|
|
2026
|
+
continue;
|
|
2027
|
+
}
|
|
2028
|
+
} else {
|
|
1946
2029
|
textBuffer += htmlChunk[i++];
|
|
1947
2030
|
continue;
|
|
1948
2031
|
}
|
|
1949
|
-
} else {
|
|
1950
|
-
textBuffer += htmlChunk[i++];
|
|
1951
|
-
continue;
|
|
1952
2032
|
}
|
|
1953
2033
|
if (nextCharCode === EXCLAMATION_CHAR) {
|
|
1954
2034
|
if (htmlChunk.charCodeAt(i + 2) === OPEN_BRACKET_CHAR) {
|
|
@@ -2454,37 +2534,41 @@ function processOpeningTag(tagName, tagId, htmlChunk, i, state, handleEvent) {
|
|
|
2454
2534
|
const headNode = state.currentNode;
|
|
2455
2535
|
if (headNode && headNode.tagId === 1) closeNode(headNode, state, handleEvent);
|
|
2456
2536
|
}
|
|
2457
|
-
if (tagId >= 0 && tagId < 113 && NEEDS_IMPLIED_END_RECOVERY[tagId] === 1)
|
|
2458
|
-
if (
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
if (
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
if (
|
|
2473
|
-
|
|
2474
|
-
if (
|
|
2475
|
-
if (parserTagDepth(state,
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
if (
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2537
|
+
if (tagId >= 0 && tagId < 113 && NEEDS_IMPLIED_END_RECOVERY[tagId] === 1) {
|
|
2538
|
+
if (tagId === 65 && parserTagDepth(state, 65) > 0) {
|
|
2539
|
+
if (closeSelectTo(state, 65, handleEvent)) return {
|
|
2540
|
+
complete: true,
|
|
2541
|
+
newPosition: result.newPosition,
|
|
2542
|
+
remainingText: "",
|
|
2543
|
+
selfClosing: false,
|
|
2544
|
+
skip: true
|
|
2545
|
+
};
|
|
2546
|
+
} else if (tagId === 67) {
|
|
2547
|
+
if (parserTagDepth(state, 65) > 0) closeSelectTo(state, 67, handleEvent);
|
|
2548
|
+
else if (parserLastTagId(state) === 67) popParserTop(state, handleEvent);
|
|
2549
|
+
} else if (tagId === 109 && parserTagDepth(state, 65) > 0) {
|
|
2550
|
+
closeSelectTo(state, 67, handleEvent);
|
|
2551
|
+
closeSelectTo(state, 109, handleEvent);
|
|
2552
|
+
} else if (tagId === 26) {
|
|
2553
|
+
if (parserTagDepth(state, 26) > 0) closeImpliedTo(state, SINGLE_A, A_SCOPE_BOUNDARY, handleEvent);
|
|
2554
|
+
} else if (tagId === 32 || tagId === 31 || tagId === 30 || tagId === 29 || tagId === 38 || tagId === 39) {
|
|
2555
|
+
if (parserTagDepth(state, 28) > 0) {
|
|
2556
|
+
if (tagId === 32 || tagId === 31) {
|
|
2557
|
+
if (parserTagDepth(state, 32) > 0 || parserTagDepth(state, 31) > 0) closeImpliedTo(state, TD_TH, CELL_SCOPE_BOUNDARY, handleEvent);
|
|
2558
|
+
} else if (tagId === 30) {
|
|
2559
|
+
if (parserTagDepth(state, 30) > 0) closeTableContext(state, TR_CELLS, handleEvent);
|
|
2560
|
+
} else closeTableContext(state, SECTION_CELLS, handleEvent);
|
|
2561
|
+
}
|
|
2562
|
+
} else {
|
|
2563
|
+
if (parserTagDepth(state, 35) > 0) closeImpliedTo(state, SINGLE_P, P_SCOPE_BOUNDARY, handleEvent);
|
|
2564
|
+
if (HEADINGS.has(tagId)) {
|
|
2565
|
+
const topId = parserLastTagId(state);
|
|
2566
|
+
if (topId !== void 0 && HEADINGS.has(topId)) popParserTop(state, handleEvent);
|
|
2567
|
+
} else if (tagId === 25) {
|
|
2568
|
+
if (parserTagDepth(state, 25) > 0) closeImpliedTo(state, SINGLE_LI, LI_SCOPE_BOUNDARY, handleEvent);
|
|
2569
|
+
} else if (tagId === 99 || tagId === 98) {
|
|
2570
|
+
if (parserTagDepth(state, 99) > 0 || parserTagDepth(state, 98) > 0) closeImpliedTo(state, DT_DD, DL_SCOPE_BOUNDARY, handleEvent);
|
|
2571
|
+
}
|
|
2488
2572
|
}
|
|
2489
2573
|
}
|
|
2490
2574
|
if (state.flattenedTagName && !result.selfClosing) {
|
|
@@ -2618,12 +2702,10 @@ function scanAttributes(source, position, selfClosing, stopAtTagEnd) {
|
|
|
2618
2702
|
state = ATTRIBUTE_GAP;
|
|
2619
2703
|
}
|
|
2620
2704
|
break;
|
|
2621
|
-
case ATTRIBUTE_UNQUOTED_VALUE:
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
}
|
|
2626
|
-
break;
|
|
2705
|
+
case ATTRIBUTE_UNQUOTED_VALUE: if (isSpace || !stopAtTagEnd && charCode === GT_CHAR) {
|
|
2706
|
+
attributes = setAttribute(attributes, source.substring(nameStart, nameEnd), source.substring(valueStart, i));
|
|
2707
|
+
state = ATTRIBUTE_GAP;
|
|
2708
|
+
}
|
|
2627
2709
|
}
|
|
2628
2710
|
i++;
|
|
2629
2711
|
}
|
|
@@ -2640,9 +2722,7 @@ function scanAttributes(source, position, selfClosing, stopAtTagEnd) {
|
|
|
2640
2722
|
attributes = setAttribute(attributes, source.substring(nameStart, nameEnd), "");
|
|
2641
2723
|
break;
|
|
2642
2724
|
case ATTRIBUTE_QUOTED_VALUE:
|
|
2643
|
-
case ATTRIBUTE_UNQUOTED_VALUE:
|
|
2644
|
-
attributes = setAttribute(attributes, source.substring(nameStart, nameEnd), source.substring(valueStart, valueEnd));
|
|
2645
|
-
break;
|
|
2725
|
+
case ATTRIBUTE_UNQUOTED_VALUE: attributes = setAttribute(attributes, source.substring(nameStart, nameEnd), source.substring(valueStart, valueEnd));
|
|
2646
2726
|
}
|
|
2647
2727
|
return {
|
|
2648
2728
|
_tag: "complete",
|
|
@@ -2660,4 +2740,4 @@ function parseAttributes(attrStr) {
|
|
|
2660
2740
|
if (result._tag === "incomplete") throw new Error("Isolated attribute scans always complete at EOF");
|
|
2661
2741
|
return result.attributes === EMPTY_ATTRIBUTES ? {} : result.attributes;
|
|
2662
2742
|
}
|
|
2663
|
-
export { blockOpenPrefix, breakHandler, buildTagOverrideHandlers, continuationPrefix, finalizeParse, getLanguageFromClass, isCharacterReferenceTail, isEmptyLinkHref, isInsideHeading, isInsideTableCell, isSafeHtmlUrl, listMarkerLineStart, orderedItemNumber, parseAttributes, parseHtml, parseHtmlStream, parseUnsignedInteger, renderBreak, resolveUrl };
|
|
2743
|
+
export { blockOpenPrefix, breakHandler, buildTagOverrideHandlers, continuationPrefix, escapeHtml, figcaptionOwnsBlockSpacing, finalizeParse, getLanguageFromClass, isCharacterReferenceTail, isEmptyLinkHref, isInsideHeading, isInsideTableCell, isSafeHtmlUrl, lastOutputChar, listMarkerLineStart, markRenderedChildContent, orderedItemNumber, parseAttributes, parseHtml, parseHtmlStream, parseUnsignedInteger, renderBreak, resolveUrl, safeAnchorOutput };
|
package/dist/_chunks/stream.mjs
CHANGED
|
@@ -19,13 +19,15 @@ async function* streamHtmlToMarkdown(htmlStream, options = {}, resolvedPlugins =
|
|
|
19
19
|
const { done, value } = await reader.read();
|
|
20
20
|
if (done) break;
|
|
21
21
|
const decoded = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
|
|
22
|
-
|
|
22
|
+
const htmlContent = `${remainingHtml}${decoded}`;
|
|
23
|
+
remainingHtml = parseHtmlStream(htmlContent, parseState, handleEvent);
|
|
23
24
|
const chunk = processor.getMarkdownChunk();
|
|
24
25
|
if (chunk) yield chunk;
|
|
25
26
|
}
|
|
26
27
|
const decoderTail = decoder.decode();
|
|
27
28
|
const finalHtml = remainingHtml + decoderTail;
|
|
28
|
-
|
|
29
|
+
const leftover = finalHtml ? parseHtmlStream(finalHtml, parseState, handleEvent) : "";
|
|
30
|
+
finalizeParse(leftover, parseState, handleEvent);
|
|
29
31
|
const finalChunk = processor.getMarkdownChunk();
|
|
30
32
|
if (finalChunk) yield finalChunk;
|
|
31
33
|
} finally {
|
package/dist/_chunks/types.d.mts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
declare const TAG_H1 = 7;
|
|
2
|
-
declare const TAG_H2 = 8;
|
|
3
|
-
declare const TAG_H3 = 9;
|
|
4
|
-
declare const TAG_H4 = 10;
|
|
5
|
-
declare const TAG_H5 = 11;
|
|
6
|
-
declare const TAG_H6 = 12;
|
|
7
|
-
declare const ELEMENT_NODE = 1;
|
|
8
|
-
declare const TEXT_NODE = 2;
|
|
1
|
+
export declare const TAG_H1 = 7;
|
|
2
|
+
export declare const TAG_H2 = 8;
|
|
3
|
+
export declare const TAG_H3 = 9;
|
|
4
|
+
export declare const TAG_H4 = 10;
|
|
5
|
+
export declare const TAG_H5 = 11;
|
|
6
|
+
export declare const TAG_H6 = 12;
|
|
7
|
+
export declare const ELEMENT_NODE = 1;
|
|
8
|
+
export declare const TEXT_NODE = 2;
|
|
9
9
|
declare const NodeEventEnter$1 = 0;
|
|
10
10
|
declare const NodeEventExit$1 = 1;
|
|
11
11
|
/**
|
|
@@ -13,7 +13,7 @@ declare const NodeEventExit$1 = 1;
|
|
|
13
13
|
* When transforms are provided, the JS engine is used regardless of engine selection.
|
|
14
14
|
* For declarative config that works with both engines, use `BuiltinPlugins`.
|
|
15
15
|
*/
|
|
16
|
-
interface TransformPlugin {
|
|
16
|
+
export interface TransformPlugin {
|
|
17
17
|
/**
|
|
18
18
|
* Whether a flattened element starts an output-excluded subtree.
|
|
19
19
|
* @internal
|
|
@@ -51,7 +51,7 @@ interface TransformPlugin {
|
|
|
51
51
|
* Declarative tag override configuration.
|
|
52
52
|
* When a string value is provided, it acts as an alias (e.g. `{ "x-heading": "h2" }`).
|
|
53
53
|
*/
|
|
54
|
-
interface TagOverride {
|
|
54
|
+
export interface TagOverride {
|
|
55
55
|
enter?: string;
|
|
56
56
|
exit?: string;
|
|
57
57
|
spacing?: [number, number];
|
|
@@ -62,7 +62,7 @@ interface TagOverride {
|
|
|
62
62
|
/**
|
|
63
63
|
* Frontmatter configuration options.
|
|
64
64
|
*/
|
|
65
|
-
interface FrontmatterConfig {
|
|
65
|
+
export interface FrontmatterConfig {
|
|
66
66
|
additionalFields?: Record<string, string>;
|
|
67
67
|
metaFields?: string[];
|
|
68
68
|
/**
|
|
@@ -75,7 +75,7 @@ interface FrontmatterConfig {
|
|
|
75
75
|
* Declarative configuration for built-in plugins.
|
|
76
76
|
* Works with both the JavaScript and Rust engines.
|
|
77
77
|
*/
|
|
78
|
-
interface BuiltinPlugins {
|
|
78
|
+
export interface BuiltinPlugins {
|
|
79
79
|
/** Filter elements by CSS selectors, tag names, or TAG_* constants */
|
|
80
80
|
filter?: {
|
|
81
81
|
include?: (string | number)[];
|
|
@@ -109,7 +109,7 @@ interface BuiltinPlugins {
|
|
|
109
109
|
* Shared engine options that work with both JS and Rust engines.
|
|
110
110
|
* This is the contract that `MarkdownEngine` methods accept.
|
|
111
111
|
*/
|
|
112
|
-
interface CleanOptions {
|
|
112
|
+
export interface CleanOptions {
|
|
113
113
|
/** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
|
|
114
114
|
urls?: boolean;
|
|
115
115
|
/** Strip fragment-only links that don't match any heading in the output */
|
|
@@ -127,7 +127,7 @@ interface CleanOptions {
|
|
|
127
127
|
/** Drop links that produce no visible text: [](url) → nothing */
|
|
128
128
|
emptyLinkText?: boolean;
|
|
129
129
|
}
|
|
130
|
-
interface EngineOptions {
|
|
130
|
+
export interface EngineOptions {
|
|
131
131
|
/**
|
|
132
132
|
* Origin URL for resolving relative image paths and internal links.
|
|
133
133
|
*/
|
|
@@ -155,7 +155,7 @@ interface EngineOptions {
|
|
|
155
155
|
*/
|
|
156
156
|
format?: 'markdown' | 'text' | 'html';
|
|
157
157
|
}
|
|
158
|
-
interface ElementNode extends Node {
|
|
158
|
+
export interface ElementNode extends Node {
|
|
159
159
|
/** Element tag name (for ELEMENT_NODE) */
|
|
160
160
|
name: string;
|
|
161
161
|
/** HTML attributes (for ELEMENT_NODE) */
|
|
@@ -169,7 +169,7 @@ interface ElementNode extends Node {
|
|
|
169
169
|
/** Plugin outputs collected during processing */
|
|
170
170
|
pluginOutput?: string[];
|
|
171
171
|
}
|
|
172
|
-
interface TextNode extends Node {
|
|
172
|
+
export interface TextNode extends Node {
|
|
173
173
|
/** Text content (for TEXT_NODE) */
|
|
174
174
|
value: string;
|
|
175
175
|
/** Markdown delimiters generated by a plugin around this literal text. */
|
|
@@ -184,7 +184,7 @@ interface TextNode extends Node {
|
|
|
184
184
|
* Base DOM node interface
|
|
185
185
|
* Optimized for streaming HTML parsing with minimal memory footprint
|
|
186
186
|
*/
|
|
187
|
-
interface Node {
|
|
187
|
+
export interface Node {
|
|
188
188
|
/** Node type (ELEMENT_NODE or TEXT_NODE) */
|
|
189
189
|
type: number;
|
|
190
190
|
/** Current nesting depth in the DOM tree */
|
|
@@ -194,7 +194,7 @@ interface Node {
|
|
|
194
194
|
index: number;
|
|
195
195
|
/** Current walk index for child traversal during streaming */
|
|
196
196
|
currentWalkIndex?: number;
|
|
197
|
-
/**
|
|
197
|
+
/** Child-content counter used by whitespace and empty-link handling */
|
|
198
198
|
childTextNodeIndex?: number;
|
|
199
199
|
/** Whether node contains whitespace - used for whitespace optimization */
|
|
200
200
|
containsWhitespace?: boolean;
|
|
@@ -250,7 +250,7 @@ interface MdreamProcessingState {
|
|
|
250
250
|
* Runtime state for markdown generation
|
|
251
251
|
* Extended state that includes output tracking and options
|
|
252
252
|
*/
|
|
253
|
-
interface MdreamRuntimeState extends Partial<MdreamProcessingState> {
|
|
253
|
+
export interface MdreamRuntimeState extends Partial<MdreamProcessingState> {
|
|
254
254
|
/** Number of newlines at end of most recent output */
|
|
255
255
|
lastNewLines?: number;
|
|
256
256
|
/** Configuration options for conversion */
|
|
@@ -296,7 +296,7 @@ type NodeEventExit = 1;
|
|
|
296
296
|
* Node event for DOM traversal
|
|
297
297
|
* Used in the event-based traversal system for streaming processing
|
|
298
298
|
*/
|
|
299
|
-
interface NodeEvent {
|
|
299
|
+
export interface NodeEvent {
|
|
300
300
|
/** Event type - enter (start tag) or exit (end tag) */
|
|
301
301
|
type: NodeEventEnter | NodeEventExit;
|
|
302
302
|
/** The node being processed */
|
|
@@ -340,7 +340,7 @@ type TagHandlerResult = string | GfmAction | undefined | void;
|
|
|
340
340
|
* Tag handler interface for HTML elements
|
|
341
341
|
* Used by plugins to extend or customize tag handling
|
|
342
342
|
*/
|
|
343
|
-
interface TagHandler {
|
|
343
|
+
export interface TagHandler {
|
|
344
344
|
enter?: (context: HandlerContext) => TagHandlerResult;
|
|
345
345
|
exit?: (context: HandlerContext) => TagHandlerResult;
|
|
346
346
|
isSelfClosing?: boolean;
|
|
@@ -368,7 +368,7 @@ interface TailwindContext {
|
|
|
368
368
|
prefix?: string;
|
|
369
369
|
suffix?: string;
|
|
370
370
|
}
|
|
371
|
-
interface PluginContext {
|
|
371
|
+
export interface PluginContext {
|
|
372
372
|
score?: number;
|
|
373
373
|
tagCount?: number;
|
|
374
374
|
linkTextLength?: number;
|
|
@@ -380,7 +380,7 @@ interface PluginContext {
|
|
|
380
380
|
/**
|
|
381
381
|
* Element extracted during conversion by the extraction plugin.
|
|
382
382
|
*/
|
|
383
|
-
interface ExtractedElement {
|
|
383
|
+
export interface ExtractedElement {
|
|
384
384
|
/** The CSS selector that matched this element */
|
|
385
385
|
selector: string;
|
|
386
386
|
/** The HTML tag name */
|
|
@@ -394,7 +394,7 @@ interface ExtractedElement {
|
|
|
394
394
|
* Top-level options for the mdream JS engine.
|
|
395
395
|
* Extends the shared `EngineOptions` with JS-specific concerns.
|
|
396
396
|
*/
|
|
397
|
-
interface MdreamOptions extends EngineOptions {
|
|
397
|
+
export interface MdreamOptions extends EngineOptions {
|
|
398
398
|
/**
|
|
399
399
|
* Imperative hook-based transform plugins.
|
|
400
400
|
* When provided, a new JS engine is created with these hooks.
|
|
@@ -405,7 +405,7 @@ interface MdreamOptions extends EngineOptions {
|
|
|
405
405
|
* Markdown chunk with content and metadata
|
|
406
406
|
* Compatible with LangChain Document structure
|
|
407
407
|
*/
|
|
408
|
-
interface MarkdownChunk {
|
|
408
|
+
export interface MarkdownChunk {
|
|
409
409
|
/** The markdown content of the chunk */
|
|
410
410
|
content: string;
|
|
411
411
|
/** Metadata extracted during chunking */
|
|
@@ -427,7 +427,7 @@ interface MarkdownChunk {
|
|
|
427
427
|
* Options for HTML to Markdown chunking
|
|
428
428
|
* Extends EngineOptions with chunking-specific settings
|
|
429
429
|
*/
|
|
430
|
-
interface SplitterOptions extends EngineOptions {
|
|
430
|
+
export interface SplitterOptions extends EngineOptions {
|
|
431
431
|
/**
|
|
432
432
|
* Header tag IDs to split on (TAG_H1, TAG_H2, etc.)
|
|
433
433
|
* @example [TAG_H1, TAG_H2]
|
|
@@ -466,4 +466,4 @@ interface SplitterOptions extends EngineOptions {
|
|
|
466
466
|
*/
|
|
467
467
|
keepSeparator?: boolean;
|
|
468
468
|
}
|
|
469
|
-
export {
|
|
469
|
+
export { NodeEventEnter$1 as NodeEventEnter, NodeEventExit$1 as NodeEventExit };
|
package/dist/cli.d.mts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export {}
|
package/dist/cli.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { streamHtmlToMarkdown } from "./_chunks/src.mjs";
|
|
2
1
|
import { withMinimalPreset } from "./preset/minimal.mjs";
|
|
2
|
+
import { streamHtmlToMarkdown } from "./index.mjs";
|
|
3
3
|
import { readFileSync } from "node:fs";
|
|
4
4
|
import { Readable } from "node:stream";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
@@ -21,7 +21,8 @@ async function streamingConvert(options = {}) {
|
|
|
21
21
|
const markdownGenerator = streamHtmlToMarkdown(Readable.toWeb(process.stdin), conversionOptions);
|
|
22
22
|
for await (const markdownChunk of markdownGenerator) if (markdownChunk && markdownChunk.length > 0) process.stdout.write(markdownChunk);
|
|
23
23
|
}
|
|
24
|
-
const
|
|
24
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
25
|
+
const packageJsonPath = join(__dirname, "..", "package.json");
|
|
25
26
|
const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
|
|
26
27
|
const cli = cac();
|
|
27
28
|
cli.command("[options]", "Convert HTML from stdin to Markdown on stdout (JS engine)").option("--origin <url>", "Origin URL for resolving relative image paths").option("--preset <preset>", "Conversion presets: minimal").option("--wrap-width <n>", "Hard-wrap prose at <n> characters on word boundaries").option("--format <format>", "Output format: markdown, text, html").option("--text", "Alias for --format text").action(async (_, opts) => {
|
package/dist/core.d.mts
CHANGED
|
@@ -1,16 +1,15 @@
|
|
|
1
1
|
import { BuiltinPlugins, MdreamOptions, TransformPlugin } from "./_chunks/types.mjs";
|
|
2
2
|
/** Imperative plugin used by the tree-shakable converter. */
|
|
3
|
-
type Plugin = TransformPlugin;
|
|
3
|
+
export type Plugin = TransformPlugin;
|
|
4
4
|
/**
|
|
5
5
|
* Options for the tree-shakable conversion entry. Import plugin factories
|
|
6
6
|
* explicitly and pass their results as an array, matching the pre-v1 JS API.
|
|
7
7
|
* Declarative built-in plugin config and cleanup live on the package root.
|
|
8
8
|
*/
|
|
9
|
-
type CoreOptions = Omit<MdreamOptions, 'plugins' | 'clean'> & {
|
|
9
|
+
export type CoreOptions = Omit<MdreamOptions, 'plugins' | 'clean'> & {
|
|
10
10
|
plugins?: Plugin[] | Pick<BuiltinPlugins, 'tagOverrides'>;
|
|
11
11
|
};
|
|
12
12
|
/** Convert HTML without bundling the optional declarative built-in plugins. */
|
|
13
|
-
declare function htmlToMarkdown(html: string, options?: CoreOptions): string;
|
|
13
|
+
export declare function htmlToMarkdown(html: string, options?: CoreOptions): string;
|
|
14
14
|
/** Stream HTML conversion without bundling declarative built-in plugins. */
|
|
15
|
-
declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: CoreOptions): AsyncIterable<string>;
|
|
16
|
-
export { CoreOptions, Plugin, htmlToMarkdown, streamHtmlToMarkdown };
|
|
15
|
+
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: CoreOptions): AsyncIterable<string>;
|
package/dist/core.mjs
CHANGED
|
@@ -14,13 +14,15 @@ function resolveTagOverrides(options) {
|
|
|
14
14
|
function htmlToMarkdown(html, options = {}) {
|
|
15
15
|
const plugins = resolveHooks(options);
|
|
16
16
|
const tagOverrides = resolveTagOverrides(options);
|
|
17
|
-
const
|
|
17
|
+
const tagOverrideHandlers = tagOverrides ? buildTagOverrideHandlers(tagOverrides) : void 0;
|
|
18
|
+
const processor = createMarkdownProcessor(options, plugins, tagOverrideHandlers);
|
|
18
19
|
processor.processHtml(html);
|
|
19
20
|
return processor.getMarkdown();
|
|
20
21
|
}
|
|
21
22
|
function streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
22
23
|
const plugins = resolveHooks(options);
|
|
23
24
|
const tagOverrides = resolveTagOverrides(options);
|
|
24
|
-
|
|
25
|
+
const tagOverrideHandlers = tagOverrides ? buildTagOverrideHandlers(tagOverrides) : void 0;
|
|
26
|
+
return streamHtmlToMarkdown$1(htmlStream, options, plugins, tagOverrideHandlers);
|
|
25
27
|
}
|
|
26
28
|
export { htmlToMarkdown, streamHtmlToMarkdown };
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,34 @@
|
|
|
1
1
|
import { ELEMENT_NODE, NodeEventEnter, NodeEventExit, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE } from "./_chunks/const.mjs";
|
|
2
|
-
import {
|
|
2
|
+
import { buildTagOverrideHandlers } from "./_chunks/parse.mjs";
|
|
3
|
+
import { applyClean, createMarkdownProcessor, resolveClean } from "./_chunks/markdown-processor.mjs";
|
|
3
4
|
import { createPlugin } from "./_chunks/plugins.mjs";
|
|
5
|
+
import { resolvePlugins } from "./_chunks/resolve-plugins.mjs";
|
|
6
|
+
import { streamHtmlToMarkdown as streamHtmlToMarkdown$1 } from "./_chunks/stream.mjs";
|
|
4
7
|
import { withMinimalPreset } from "./preset/minimal.mjs";
|
|
8
|
+
function resolveHooks(options) {
|
|
9
|
+
return options.hooks?.length ? options.hooks : void 0;
|
|
10
|
+
}
|
|
11
|
+
function convert(html, options, hooks) {
|
|
12
|
+
const { plugins, callExtractionHandlers, getFrontmatter, frontmatterCallback } = resolvePlugins(options, hooks);
|
|
13
|
+
const tagOverrideHandlers = options.plugins?.tagOverrides ? buildTagOverrideHandlers(options.plugins.tagOverrides) : void 0;
|
|
14
|
+
const processor = createMarkdownProcessor(options, plugins, tagOverrideHandlers);
|
|
15
|
+
processor.processHtml(html);
|
|
16
|
+
if (getFrontmatter && frontmatterCallback) {
|
|
17
|
+
const fm = getFrontmatter();
|
|
18
|
+
if (fm) frontmatterCallback(fm);
|
|
19
|
+
}
|
|
20
|
+
callExtractionHandlers?.();
|
|
21
|
+
return processor.getMarkdown();
|
|
22
|
+
}
|
|
23
|
+
function htmlToMarkdown(html, options = {}) {
|
|
24
|
+
const markdown = convert(html, options, resolveHooks(options));
|
|
25
|
+
if (options.clean && (options.format === void 0 || options.format === "markdown")) return applyClean(markdown, resolveClean(options.clean));
|
|
26
|
+
return markdown;
|
|
27
|
+
}
|
|
28
|
+
function streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
29
|
+
const hooks = resolveHooks(options);
|
|
30
|
+
const { plugins } = resolvePlugins(options, hooks);
|
|
31
|
+
const tagOverrideHandlers = options.plugins?.tagOverrides ? buildTagOverrideHandlers(options.plugins.tagOverrides) : void 0;
|
|
32
|
+
return streamHtmlToMarkdown$1(htmlStream, options, plugins, tagOverrideHandlers);
|
|
33
|
+
}
|
|
5
34
|
export { ELEMENT_NODE, NodeEventEnter, NodeEventExit, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, createPlugin, htmlToMarkdown, streamHtmlToMarkdown, withMinimalPreset };
|