@stream-mdx/core 0.0.2 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/index.cjs +358 -27
- package/dist/index.d.cts +3 -2
- package/dist/index.d.ts +3 -2
- package/dist/index.mjs +356 -27
- package/dist/inline-parser.cjs +100 -18
- package/dist/inline-parser.d.cts +10 -0
- package/dist/inline-parser.d.ts +10 -0
- package/dist/inline-parser.mjs +100 -18
- package/dist/mixed-content.cjs +130 -9
- package/dist/mixed-content.d.cts +16 -2
- package/dist/mixed-content.d.ts +16 -2
- package/dist/mixed-content.mjs +130 -9
- package/dist/streaming/inline-streaming.cjs +153 -0
- package/dist/streaming/inline-streaming.d.cts +28 -0
- package/dist/streaming/inline-streaming.d.ts +28 -0
- package/dist/streaming/inline-streaming.mjs +127 -0
- package/dist/types.d.cts +24 -1
- package/dist/types.d.ts +24 -1
- package/dist/worker-html-sanitizer.cjs +16 -2
- package/dist/worker-html-sanitizer.mjs +16 -2
- package/package.json +7 -2
package/dist/index.mjs
CHANGED
|
@@ -202,12 +202,42 @@ function filterAllowedAttributes(attrs) {
|
|
|
202
202
|
}
|
|
203
203
|
|
|
204
204
|
// src/inline-parser.ts
|
|
205
|
+
function ensureGlobal(pattern) {
|
|
206
|
+
if (pattern.global) return pattern;
|
|
207
|
+
const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`;
|
|
208
|
+
return new RegExp(pattern.source, flags);
|
|
209
|
+
}
|
|
210
|
+
function findLastMatch(pattern, value) {
|
|
211
|
+
const re = ensureGlobal(pattern);
|
|
212
|
+
let match = null;
|
|
213
|
+
let next;
|
|
214
|
+
while ((next = re.exec(value)) !== null) {
|
|
215
|
+
match = next;
|
|
216
|
+
}
|
|
217
|
+
return match;
|
|
218
|
+
}
|
|
219
|
+
function findMatchAfter(pattern, value, startIndex) {
|
|
220
|
+
const re = ensureGlobal(pattern);
|
|
221
|
+
re.lastIndex = Math.max(0, startIndex);
|
|
222
|
+
return re.exec(value);
|
|
223
|
+
}
|
|
224
|
+
function isSamePattern(a, b) {
|
|
225
|
+
return a.source === b.source && a.flags === b.flags;
|
|
226
|
+
}
|
|
227
|
+
function countMatches(pattern, value) {
|
|
228
|
+
const re = ensureGlobal(pattern);
|
|
229
|
+
let count = 0;
|
|
230
|
+
while (re.exec(value)) {
|
|
231
|
+
count += 1;
|
|
232
|
+
}
|
|
233
|
+
return count;
|
|
234
|
+
}
|
|
205
235
|
var InlineParser = class {
|
|
206
236
|
constructor(options = {}) {
|
|
207
237
|
this.plugins = [];
|
|
208
238
|
this.cache = /* @__PURE__ */ new Map();
|
|
209
239
|
this.maxCacheEntries = Number.isFinite(options.maxCacheEntries ?? Number.NaN) ? Math.max(0, options.maxCacheEntries ?? 0) : 2e3;
|
|
210
|
-
this.registerDefaultPlugins();
|
|
240
|
+
this.registerDefaultPlugins({ enableMath: options.enableMath !== false });
|
|
211
241
|
}
|
|
212
242
|
/**
|
|
213
243
|
* Register a plugin with the parser
|
|
@@ -249,6 +279,42 @@ var InlineParser = class {
|
|
|
249
279
|
}
|
|
250
280
|
return result;
|
|
251
281
|
}
|
|
282
|
+
/**
|
|
283
|
+
* Streaming regex anticipation helper. Returns an append string if a plugin
|
|
284
|
+
* declares an incomplete match at the end of the buffer.
|
|
285
|
+
*/
|
|
286
|
+
getRegexAnticipationAppend(content) {
|
|
287
|
+
if (!content || this.plugins.length === 0) {
|
|
288
|
+
return null;
|
|
289
|
+
}
|
|
290
|
+
for (const plugin of this.plugins) {
|
|
291
|
+
if (!("re" in plugin)) continue;
|
|
292
|
+
const regexPlugin = plugin;
|
|
293
|
+
const anticipation = regexPlugin.anticipation;
|
|
294
|
+
if (!anticipation) continue;
|
|
295
|
+
const maxScanChars = Number.isFinite(anticipation.maxScanChars ?? Number.NaN) ? Math.max(1, anticipation.maxScanChars ?? 0) : 240;
|
|
296
|
+
const scan = content.slice(Math.max(0, content.length - maxScanChars));
|
|
297
|
+
if (regexPlugin.fastCheck && !regexPlugin.fastCheck(scan)) {
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
300
|
+
const lastStart = findLastMatch(anticipation.start, scan);
|
|
301
|
+
if (!lastStart) continue;
|
|
302
|
+
if (isSamePattern(anticipation.start, anticipation.end)) {
|
|
303
|
+
const occurrences = countMatches(anticipation.start, scan);
|
|
304
|
+
if (occurrences % 2 === 0) {
|
|
305
|
+
continue;
|
|
306
|
+
}
|
|
307
|
+
} else {
|
|
308
|
+
const startIndex = lastStart.index + lastStart[0].length;
|
|
309
|
+
const hasEnd = Boolean(findMatchAfter(anticipation.end, scan, startIndex));
|
|
310
|
+
if (hasEnd) continue;
|
|
311
|
+
}
|
|
312
|
+
const appendValue = typeof anticipation.append === "function" ? anticipation.append(lastStart, content) : anticipation.append;
|
|
313
|
+
if (!appendValue) continue;
|
|
314
|
+
return appendValue;
|
|
315
|
+
}
|
|
316
|
+
return null;
|
|
317
|
+
}
|
|
252
318
|
/**
|
|
253
319
|
* Clear the memoization cache
|
|
254
320
|
*/
|
|
@@ -273,10 +339,27 @@ var InlineParser = class {
|
|
|
273
339
|
* Register default plugins with proper precedence ordering
|
|
274
340
|
* Lower priority numbers = higher precedence (run first)
|
|
275
341
|
*/
|
|
276
|
-
registerDefaultPlugins() {
|
|
342
|
+
registerDefaultPlugins(options) {
|
|
343
|
+
if (options.enableMath) {
|
|
344
|
+
this.registerPlugin({
|
|
345
|
+
id: "math-display",
|
|
346
|
+
priority: 0,
|
|
347
|
+
re: /\$\$([\s\S]+?)\$\$/g,
|
|
348
|
+
toNode: (match) => ({ kind: "math-display", tex: match[1].trim() }),
|
|
349
|
+
fastCheck: (text) => text.indexOf("$$") !== -1
|
|
350
|
+
});
|
|
351
|
+
this.registerPlugin({
|
|
352
|
+
id: "math-inline",
|
|
353
|
+
priority: 1,
|
|
354
|
+
re: /\$([^$\n]+?)\$/g,
|
|
355
|
+
// Non-greedy to prevent spanning multiple expressions
|
|
356
|
+
toNode: (match) => ({ kind: "math-inline", tex: match[1].trim() }),
|
|
357
|
+
fastCheck: (text) => text.indexOf("$") !== -1
|
|
358
|
+
});
|
|
359
|
+
}
|
|
277
360
|
this.registerPlugin({
|
|
278
361
|
id: "escaped-character",
|
|
279
|
-
priority:
|
|
362
|
+
priority: 2,
|
|
280
363
|
re: /\\([\\`*_{}\[\]()#+\-.!>])/g,
|
|
281
364
|
toNode: (match) => ({
|
|
282
365
|
kind: "text",
|
|
@@ -285,19 +368,11 @@ var InlineParser = class {
|
|
|
285
368
|
fastCheck: (text) => text.indexOf("\\") !== -1
|
|
286
369
|
});
|
|
287
370
|
this.registerPlugin({
|
|
288
|
-
id: "
|
|
289
|
-
priority: 1,
|
|
290
|
-
re: /\$\$([^$]+?)\$\$/g,
|
|
291
|
-
toNode: (match) => ({ kind: "math-display", tex: match[1].trim() }),
|
|
292
|
-
fastCheck: (text) => text.indexOf("$$") !== -1
|
|
293
|
-
});
|
|
294
|
-
this.registerPlugin({
|
|
295
|
-
id: "math-inline",
|
|
371
|
+
id: "hard-break",
|
|
296
372
|
priority: 2,
|
|
297
|
-
re:
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
fastCheck: (text) => text.indexOf("$") !== -1
|
|
373
|
+
re: /\\\r?\n| {2,}\r?\n/g,
|
|
374
|
+
toNode: (_match) => ({ kind: "br" }),
|
|
375
|
+
fastCheck: (text) => text.indexOf("\n") !== -1 || text.indexOf("\r") !== -1
|
|
301
376
|
});
|
|
302
377
|
this.registerPlugin({
|
|
303
378
|
id: "code-spans",
|
|
@@ -368,9 +443,16 @@ var InlineParser = class {
|
|
|
368
443
|
this.registerPlugin({
|
|
369
444
|
id: "citations",
|
|
370
445
|
priority: 10,
|
|
371
|
-
re: /\[\^([^\]]+)\]|@cite\{([^}]+)\}/g,
|
|
372
|
-
toNode: (match) => ({ kind: "citation", id: match[1] || match[2] }),
|
|
373
|
-
fastCheck: (text) => text.indexOf("@") !== -1 || text.indexOf("[^") !== -1
|
|
446
|
+
re: /\[\^([^\]\n]+)\]|@cite\{([^}\n]+)\}|\{cite:([^}\n]+)\}/g,
|
|
447
|
+
toNode: (match) => ({ kind: "citation", id: match[1] || match[2] || match[3] }),
|
|
448
|
+
fastCheck: (text) => text.indexOf("@cite") !== -1 || text.indexOf("[^") !== -1 || text.indexOf("{cite:") !== -1,
|
|
449
|
+
anticipation: {
|
|
450
|
+
start: /@cite\{|\{cite:/g,
|
|
451
|
+
end: /\}/g,
|
|
452
|
+
full: /@cite\{[^}\n]+?\}|\{cite:[^}\n]+?\}/g,
|
|
453
|
+
append: "}",
|
|
454
|
+
maxScanChars: 120
|
|
455
|
+
}
|
|
374
456
|
});
|
|
375
457
|
this.registerPlugin({
|
|
376
458
|
id: "mentions",
|
|
@@ -470,9 +552,23 @@ import * as rehypeParse from "rehype-parse";
|
|
|
470
552
|
import * as rehypeSanitize from "rehype-sanitize";
|
|
471
553
|
import * as rehypeStringify from "rehype-stringify";
|
|
472
554
|
import { unified } from "unified";
|
|
473
|
-
var
|
|
555
|
+
var rehypeSanitizeModule = rehypeSanitize;
|
|
556
|
+
var defaultSchema = rehypeSanitizeModule.defaultSchema;
|
|
557
|
+
var resolvePlugin = (mod) => {
|
|
558
|
+
if (typeof mod === "function") return mod;
|
|
559
|
+
if (mod && typeof mod.default === "function") {
|
|
560
|
+
return mod.default;
|
|
561
|
+
}
|
|
562
|
+
if (mod && typeof mod.default?.default === "function") {
|
|
563
|
+
return mod.default?.default;
|
|
564
|
+
}
|
|
565
|
+
return mod;
|
|
566
|
+
};
|
|
567
|
+
var rehypeParsePlugin = resolvePlugin(rehypeParse);
|
|
568
|
+
var rehypeSanitizePlugin = resolvePlugin(rehypeSanitizeModule);
|
|
569
|
+
var rehypeStringifyPlugin = resolvePlugin(rehypeStringify);
|
|
474
570
|
var SANITIZED_SCHEMA = createSchema();
|
|
475
|
-
var sanitizeProcessor = unified().use(
|
|
571
|
+
var sanitizeProcessor = unified().use(rehypeParsePlugin, { fragment: true }).use(rehypeSanitizePlugin, SANITIZED_SCHEMA).use(rehypeStringifyPlugin).freeze();
|
|
476
572
|
function sanitizeHtmlInWorker(html) {
|
|
477
573
|
if (!html) return "";
|
|
478
574
|
try {
|
|
@@ -588,9 +684,27 @@ function mergeAttributes(existing, additions) {
|
|
|
588
684
|
}
|
|
589
685
|
|
|
590
686
|
// src/mixed-content.ts
|
|
591
|
-
|
|
687
|
+
var DEFAULT_INLINE_HTML_AUTOCLOSE_TAGS = /* @__PURE__ */ new Set([
|
|
688
|
+
"span",
|
|
689
|
+
"em",
|
|
690
|
+
"strong",
|
|
691
|
+
"code",
|
|
692
|
+
"kbd",
|
|
693
|
+
"del",
|
|
694
|
+
"s",
|
|
695
|
+
"mark",
|
|
696
|
+
"sub",
|
|
697
|
+
"sup",
|
|
698
|
+
"i",
|
|
699
|
+
"b",
|
|
700
|
+
"u",
|
|
701
|
+
"small",
|
|
702
|
+
"abbr",
|
|
703
|
+
"a"
|
|
704
|
+
]);
|
|
705
|
+
function extractMixedContentSegments(raw, baseOffset, parseInline, options) {
|
|
592
706
|
if (!raw) return [];
|
|
593
|
-
const initial = splitByTagSegments(raw, baseOffset, parseInline);
|
|
707
|
+
const initial = splitByTagSegments(raw, baseOffset, parseInline, options);
|
|
594
708
|
const expanded = [];
|
|
595
709
|
for (const segment of initial) {
|
|
596
710
|
if (segment.kind === "text") {
|
|
@@ -601,22 +715,58 @@ function extractMixedContentSegments(raw, baseOffset, parseInline) {
|
|
|
601
715
|
}
|
|
602
716
|
return mergeAdjacentTextSegments(expanded, parseInline);
|
|
603
717
|
}
|
|
604
|
-
function splitByTagSegments(source, baseOffset, parseInline) {
|
|
718
|
+
function splitByTagSegments(source, baseOffset, parseInline, options) {
|
|
605
719
|
const segments = [];
|
|
606
720
|
const lowerSource = source.toLowerCase();
|
|
607
721
|
const tagPattern = /<([A-Za-z][\w:-]*)([^<>]*?)\/?>/g;
|
|
608
722
|
let cursor = 0;
|
|
609
723
|
let match = tagPattern.exec(source);
|
|
610
724
|
const baseIsFinite = typeof baseOffset === "number" && Number.isFinite(baseOffset);
|
|
725
|
+
const htmlAllowTags = normalizeHtmlAllowlist(options?.html?.allowTags);
|
|
726
|
+
const htmlAutoClose = options?.html?.autoClose === true;
|
|
727
|
+
const htmlMaxNewlines = normalizeNewlineLimit(options?.html?.maxNewlines);
|
|
728
|
+
const mdxAutoClose = options?.mdx?.autoClose === true;
|
|
729
|
+
const mdxMaxNewlines = normalizeNewlineLimit(options?.mdx?.maxNewlines);
|
|
730
|
+
const mdxAllowlist = normalizeComponentAllowlist(options?.mdx?.componentAllowlist);
|
|
611
731
|
while (match !== null) {
|
|
612
732
|
const start = match.index;
|
|
613
733
|
const tagName = match[1];
|
|
614
734
|
const matchText = match[0];
|
|
615
|
-
const
|
|
735
|
+
const tagNameLower = tagName.toLowerCase();
|
|
736
|
+
const isSelfClosing = matchText.endsWith("/>") || isVoidHtmlTag(tagNameLower);
|
|
737
|
+
const mdxCandidate = isLikelyMdxComponent(tagName);
|
|
738
|
+
const mdxAllowed = mdxCandidate && (!mdxAllowlist || mdxAllowlist.has(tagName));
|
|
739
|
+
if (mdxCandidate && mdxAllowlist && !mdxAllowed) {
|
|
740
|
+
tagPattern.lastIndex = start + 1;
|
|
741
|
+
match = tagPattern.exec(source);
|
|
742
|
+
continue;
|
|
743
|
+
}
|
|
616
744
|
let end = tagPattern.lastIndex;
|
|
617
|
-
if (!isSelfClosing && !
|
|
745
|
+
if (!isSelfClosing && !mdxAllowed) {
|
|
618
746
|
const closingIndex = findClosingHtmlTag(lowerSource, tagName.toLowerCase(), end);
|
|
619
747
|
if (closingIndex === -1) {
|
|
748
|
+
if (htmlAutoClose && htmlAllowTags.has(tagNameLower)) {
|
|
749
|
+
const tail = source.slice(end);
|
|
750
|
+
const newlineCount = countNewlines(tail, htmlMaxNewlines + 1);
|
|
751
|
+
if (newlineCount <= htmlMaxNewlines) {
|
|
752
|
+
if (start > cursor) {
|
|
753
|
+
const absoluteFrom = baseIsFinite ? baseOffset + cursor : void 0;
|
|
754
|
+
const absoluteTo = baseIsFinite ? baseOffset + start : void 0;
|
|
755
|
+
pushTextSegment(segments, source.slice(cursor, start), absoluteFrom, absoluteTo, parseInline);
|
|
756
|
+
}
|
|
757
|
+
const rawSegment2 = source.slice(start);
|
|
758
|
+
const closedValue = `${rawSegment2}</${tagName}>`;
|
|
759
|
+
const segment2 = {
|
|
760
|
+
kind: "html",
|
|
761
|
+
value: closedValue,
|
|
762
|
+
range: createSegmentRange(baseOffset, start, source.length),
|
|
763
|
+
sanitized: sanitizeHtmlInWorker(closedValue)
|
|
764
|
+
};
|
|
765
|
+
segments.push(segment2);
|
|
766
|
+
cursor = source.length;
|
|
767
|
+
break;
|
|
768
|
+
}
|
|
769
|
+
}
|
|
620
770
|
tagPattern.lastIndex = start + 1;
|
|
621
771
|
match = tagPattern.exec(source);
|
|
622
772
|
continue;
|
|
@@ -628,8 +778,8 @@ function splitByTagSegments(source, baseOffset, parseInline) {
|
|
|
628
778
|
const absoluteTo = baseIsFinite ? baseOffset + start : void 0;
|
|
629
779
|
pushTextSegment(segments, source.slice(cursor, start), absoluteFrom, absoluteTo, parseInline);
|
|
630
780
|
}
|
|
631
|
-
|
|
632
|
-
const kind =
|
|
781
|
+
let rawSegment = source.slice(start, end);
|
|
782
|
+
const kind = mdxAllowed ? "mdx" : "html";
|
|
633
783
|
const segment = {
|
|
634
784
|
kind,
|
|
635
785
|
value: rawSegment,
|
|
@@ -638,6 +788,17 @@ function splitByTagSegments(source, baseOffset, parseInline) {
|
|
|
638
788
|
if (kind === "html") {
|
|
639
789
|
segment.sanitized = sanitizeHtmlInWorker(rawSegment);
|
|
640
790
|
} else {
|
|
791
|
+
const tail = source.slice(end);
|
|
792
|
+
const newlineCount = countNewlines(tail, mdxMaxNewlines + 1);
|
|
793
|
+
if (mdxAutoClose && newlineCount > mdxMaxNewlines) {
|
|
794
|
+
tagPattern.lastIndex = start + 1;
|
|
795
|
+
match = tagPattern.exec(source);
|
|
796
|
+
continue;
|
|
797
|
+
}
|
|
798
|
+
if (mdxAutoClose && !rawSegment.endsWith("/>")) {
|
|
799
|
+
rawSegment = selfCloseTag(rawSegment);
|
|
800
|
+
segment.value = rawSegment;
|
|
801
|
+
}
|
|
641
802
|
segment.status = "pending";
|
|
642
803
|
}
|
|
643
804
|
segments.push(segment);
|
|
@@ -762,6 +923,48 @@ var VOID_HTML_TAGS = /* @__PURE__ */ new Set(["br", "hr", "img", "meta", "input"
|
|
|
762
923
|
function isVoidHtmlTag(tagName) {
|
|
763
924
|
return VOID_HTML_TAGS.has(tagName.toLowerCase());
|
|
764
925
|
}
|
|
926
|
+
function normalizeNewlineLimit(value) {
|
|
927
|
+
if (!Number.isFinite(value ?? Number.NaN)) {
|
|
928
|
+
return 2;
|
|
929
|
+
}
|
|
930
|
+
return Math.max(0, value ?? 0);
|
|
931
|
+
}
|
|
932
|
+
function normalizeHtmlAllowlist(value) {
|
|
933
|
+
if (!value) return DEFAULT_INLINE_HTML_AUTOCLOSE_TAGS;
|
|
934
|
+
const tags = /* @__PURE__ */ new Set();
|
|
935
|
+
for (const tag of value) {
|
|
936
|
+
if (tag) {
|
|
937
|
+
tags.add(tag.toLowerCase());
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
return tags.size > 0 ? tags : DEFAULT_INLINE_HTML_AUTOCLOSE_TAGS;
|
|
941
|
+
}
|
|
942
|
+
function normalizeComponentAllowlist(value) {
|
|
943
|
+
if (!value) return null;
|
|
944
|
+
const tags = /* @__PURE__ */ new Set();
|
|
945
|
+
for (const tag of value) {
|
|
946
|
+
if (tag) tags.add(tag);
|
|
947
|
+
}
|
|
948
|
+
return tags.size > 0 ? tags : null;
|
|
949
|
+
}
|
|
950
|
+
function countNewlines(value, limit) {
|
|
951
|
+
let count = 0;
|
|
952
|
+
for (let i = 0; i < value.length; i++) {
|
|
953
|
+
if (value.charCodeAt(i) === 10) {
|
|
954
|
+
count += 1;
|
|
955
|
+
if (limit !== void 0 && count >= limit) {
|
|
956
|
+
return count;
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
return count;
|
|
961
|
+
}
|
|
962
|
+
function selfCloseTag(rawTag) {
|
|
963
|
+
if (rawTag.endsWith("/>")) return rawTag;
|
|
964
|
+
const closeIndex = rawTag.lastIndexOf(">");
|
|
965
|
+
if (closeIndex === -1) return rawTag;
|
|
966
|
+
return `${rawTag.slice(0, closeIndex)}/>`;
|
|
967
|
+
}
|
|
765
968
|
function isLikelyMdxComponent(tagName) {
|
|
766
969
|
const first = tagName.charAt(0);
|
|
767
970
|
return first.toUpperCase() === first && first.toLowerCase() !== first;
|
|
@@ -2712,6 +2915,130 @@ var CustomStreamingMatcher = class {
|
|
|
2712
2915
|
return false;
|
|
2713
2916
|
}
|
|
2714
2917
|
};
|
|
2918
|
+
|
|
2919
|
+
// src/streaming/inline-streaming.ts
|
|
2920
|
+
var DEFAULT_FORMAT_ANTICIPATION = {
|
|
2921
|
+
inline: false,
|
|
2922
|
+
mathInline: false,
|
|
2923
|
+
mathBlock: false,
|
|
2924
|
+
html: false,
|
|
2925
|
+
mdx: false,
|
|
2926
|
+
regex: false
|
|
2927
|
+
};
|
|
2928
|
+
function normalizeFormatAnticipation(input) {
|
|
2929
|
+
if (input === true) {
|
|
2930
|
+
return { ...DEFAULT_FORMAT_ANTICIPATION, inline: true };
|
|
2931
|
+
}
|
|
2932
|
+
if (!input) {
|
|
2933
|
+
return { ...DEFAULT_FORMAT_ANTICIPATION };
|
|
2934
|
+
}
|
|
2935
|
+
return {
|
|
2936
|
+
inline: input.inline ?? false,
|
|
2937
|
+
mathInline: input.mathInline ?? false,
|
|
2938
|
+
mathBlock: input.mathBlock ?? false,
|
|
2939
|
+
html: input.html ?? false,
|
|
2940
|
+
mdx: input.mdx ?? false,
|
|
2941
|
+
regex: input.regex ?? false
|
|
2942
|
+
};
|
|
2943
|
+
}
|
|
2944
|
+
function prepareInlineStreamingContent(content, options) {
|
|
2945
|
+
const enableMath = options?.math !== false;
|
|
2946
|
+
const anticipation = normalizeFormatAnticipation(options?.formatAnticipation);
|
|
2947
|
+
const enableInlineAnticipation = anticipation.inline;
|
|
2948
|
+
const enableMathInlineAnticipation = anticipation.mathInline;
|
|
2949
|
+
const enableMathBlockAnticipation = anticipation.mathBlock;
|
|
2950
|
+
const stack = [];
|
|
2951
|
+
const toggleToken = (token) => {
|
|
2952
|
+
const last = stack[stack.length - 1];
|
|
2953
|
+
if (last === token) {
|
|
2954
|
+
stack.pop();
|
|
2955
|
+
} else {
|
|
2956
|
+
stack.push(token);
|
|
2957
|
+
}
|
|
2958
|
+
};
|
|
2959
|
+
let mathDisplayOpen = false;
|
|
2960
|
+
let mathDisplayCrossedNewline = false;
|
|
2961
|
+
for (let i = 0; i < content.length; i++) {
|
|
2962
|
+
const code = content.charCodeAt(i);
|
|
2963
|
+
if (code === 10 || code === 13) {
|
|
2964
|
+
if (mathDisplayOpen) {
|
|
2965
|
+
mathDisplayCrossedNewline = true;
|
|
2966
|
+
}
|
|
2967
|
+
continue;
|
|
2968
|
+
}
|
|
2969
|
+
if (code === 96) {
|
|
2970
|
+
toggleToken("code");
|
|
2971
|
+
continue;
|
|
2972
|
+
}
|
|
2973
|
+
if (code === 126 && i + 1 < content.length && content.charCodeAt(i + 1) === 126) {
|
|
2974
|
+
toggleToken("strike");
|
|
2975
|
+
i += 1;
|
|
2976
|
+
continue;
|
|
2977
|
+
}
|
|
2978
|
+
if (code === 42) {
|
|
2979
|
+
if (i + 1 < content.length && content.charCodeAt(i + 1) === 42) {
|
|
2980
|
+
toggleToken("strong");
|
|
2981
|
+
i += 1;
|
|
2982
|
+
} else {
|
|
2983
|
+
toggleToken("em");
|
|
2984
|
+
}
|
|
2985
|
+
continue;
|
|
2986
|
+
}
|
|
2987
|
+
if (enableMath && code === 36) {
|
|
2988
|
+
if (i + 1 < content.length && content.charCodeAt(i + 1) === 36) {
|
|
2989
|
+
toggleToken("math-display");
|
|
2990
|
+
if (mathDisplayOpen) {
|
|
2991
|
+
mathDisplayOpen = false;
|
|
2992
|
+
mathDisplayCrossedNewline = false;
|
|
2993
|
+
} else {
|
|
2994
|
+
mathDisplayOpen = true;
|
|
2995
|
+
mathDisplayCrossedNewline = false;
|
|
2996
|
+
}
|
|
2997
|
+
i += 1;
|
|
2998
|
+
} else {
|
|
2999
|
+
toggleToken("math-inline");
|
|
3000
|
+
}
|
|
3001
|
+
}
|
|
3002
|
+
}
|
|
3003
|
+
const hasIncompleteFormatting = stack.some((token) => token === "code" || token === "strike" || token === "strong" || token === "em");
|
|
3004
|
+
const hasIncompleteMathInline = stack.includes("math-inline");
|
|
3005
|
+
const hasIncompleteMathDisplay = stack.includes("math-display");
|
|
3006
|
+
const hasIncompleteMath = hasIncompleteMathInline || hasIncompleteMathDisplay;
|
|
3007
|
+
if (enableMath && hasIncompleteMath) {
|
|
3008
|
+
if (hasIncompleteMathInline && !enableMathInlineAnticipation) {
|
|
3009
|
+
return { kind: "raw", status: "raw", reason: "incomplete-math" };
|
|
3010
|
+
}
|
|
3011
|
+
if (hasIncompleteMathDisplay && (!enableMathBlockAnticipation || mathDisplayCrossedNewline)) {
|
|
3012
|
+
return { kind: "raw", status: "raw", reason: "incomplete-math" };
|
|
3013
|
+
}
|
|
3014
|
+
}
|
|
3015
|
+
if (hasIncompleteFormatting && !enableInlineAnticipation) {
|
|
3016
|
+
return { kind: "raw", status: "raw", reason: "incomplete-formatting" };
|
|
3017
|
+
}
|
|
3018
|
+
if (!hasIncompleteFormatting && !hasIncompleteMath) {
|
|
3019
|
+
return { kind: "parse", status: "complete", content, appended: "" };
|
|
3020
|
+
}
|
|
3021
|
+
const appendForToken = (token) => {
|
|
3022
|
+
switch (token) {
|
|
3023
|
+
case "code":
|
|
3024
|
+
return "`";
|
|
3025
|
+
case "strike":
|
|
3026
|
+
return "~~";
|
|
3027
|
+
case "strong":
|
|
3028
|
+
return "**";
|
|
3029
|
+
case "em":
|
|
3030
|
+
return "*";
|
|
3031
|
+
case "math-inline":
|
|
3032
|
+
return "$";
|
|
3033
|
+
case "math-display":
|
|
3034
|
+
return "$$";
|
|
3035
|
+
default:
|
|
3036
|
+
return "";
|
|
3037
|
+
}
|
|
3038
|
+
};
|
|
3039
|
+
const appended = stack.slice().reverse().map((token) => appendForToken(token)).join("");
|
|
3040
|
+
return { kind: "parse", status: "anticipated", content: content + appended, appended };
|
|
3041
|
+
}
|
|
2715
3042
|
export {
|
|
2716
3043
|
CSP_HEADERS,
|
|
2717
3044
|
CustomStreamingMatcher,
|
|
@@ -2755,8 +3082,10 @@ export {
|
|
|
2755
3082
|
inlineNodesToPlainText,
|
|
2756
3083
|
isLikelyMdxComponent,
|
|
2757
3084
|
normalizeBlockquoteText,
|
|
3085
|
+
normalizeFormatAnticipation,
|
|
2758
3086
|
normalizeLang,
|
|
2759
3087
|
parseCodeFenceInfo,
|
|
3088
|
+
prepareInlineStreamingContent,
|
|
2760
3089
|
removeHeadingMarkers,
|
|
2761
3090
|
sanitizeCodeHTML,
|
|
2762
3091
|
sanitizeHTML,
|
package/dist/inline-parser.cjs
CHANGED
|
@@ -25,12 +25,42 @@ __export(inline_parser_exports, {
|
|
|
25
25
|
applyRegexPlugin: () => applyRegexPlugin
|
|
26
26
|
});
|
|
27
27
|
module.exports = __toCommonJS(inline_parser_exports);
|
|
28
|
+
function ensureGlobal(pattern) {
|
|
29
|
+
if (pattern.global) return pattern;
|
|
30
|
+
const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`;
|
|
31
|
+
return new RegExp(pattern.source, flags);
|
|
32
|
+
}
|
|
33
|
+
function findLastMatch(pattern, value) {
|
|
34
|
+
const re = ensureGlobal(pattern);
|
|
35
|
+
let match = null;
|
|
36
|
+
let next;
|
|
37
|
+
while ((next = re.exec(value)) !== null) {
|
|
38
|
+
match = next;
|
|
39
|
+
}
|
|
40
|
+
return match;
|
|
41
|
+
}
|
|
42
|
+
function findMatchAfter(pattern, value, startIndex) {
|
|
43
|
+
const re = ensureGlobal(pattern);
|
|
44
|
+
re.lastIndex = Math.max(0, startIndex);
|
|
45
|
+
return re.exec(value);
|
|
46
|
+
}
|
|
47
|
+
function isSamePattern(a, b) {
|
|
48
|
+
return a.source === b.source && a.flags === b.flags;
|
|
49
|
+
}
|
|
50
|
+
function countMatches(pattern, value) {
|
|
51
|
+
const re = ensureGlobal(pattern);
|
|
52
|
+
let count = 0;
|
|
53
|
+
while (re.exec(value)) {
|
|
54
|
+
count += 1;
|
|
55
|
+
}
|
|
56
|
+
return count;
|
|
57
|
+
}
|
|
28
58
|
var InlineParser = class {
|
|
29
59
|
constructor(options = {}) {
|
|
30
60
|
this.plugins = [];
|
|
31
61
|
this.cache = /* @__PURE__ */ new Map();
|
|
32
62
|
this.maxCacheEntries = Number.isFinite(options.maxCacheEntries ?? Number.NaN) ? Math.max(0, options.maxCacheEntries ?? 0) : 2e3;
|
|
33
|
-
this.registerDefaultPlugins();
|
|
63
|
+
this.registerDefaultPlugins({ enableMath: options.enableMath !== false });
|
|
34
64
|
}
|
|
35
65
|
/**
|
|
36
66
|
* Register a plugin with the parser
|
|
@@ -72,6 +102,42 @@ var InlineParser = class {
|
|
|
72
102
|
}
|
|
73
103
|
return result;
|
|
74
104
|
}
|
|
105
|
+
/**
|
|
106
|
+
* Streaming regex anticipation helper. Returns an append string if a plugin
|
|
107
|
+
* declares an incomplete match at the end of the buffer.
|
|
108
|
+
*/
|
|
109
|
+
getRegexAnticipationAppend(content) {
|
|
110
|
+
if (!content || this.plugins.length === 0) {
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
for (const plugin of this.plugins) {
|
|
114
|
+
if (!("re" in plugin)) continue;
|
|
115
|
+
const regexPlugin = plugin;
|
|
116
|
+
const anticipation = regexPlugin.anticipation;
|
|
117
|
+
if (!anticipation) continue;
|
|
118
|
+
const maxScanChars = Number.isFinite(anticipation.maxScanChars ?? Number.NaN) ? Math.max(1, anticipation.maxScanChars ?? 0) : 240;
|
|
119
|
+
const scan = content.slice(Math.max(0, content.length - maxScanChars));
|
|
120
|
+
if (regexPlugin.fastCheck && !regexPlugin.fastCheck(scan)) {
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
const lastStart = findLastMatch(anticipation.start, scan);
|
|
124
|
+
if (!lastStart) continue;
|
|
125
|
+
if (isSamePattern(anticipation.start, anticipation.end)) {
|
|
126
|
+
const occurrences = countMatches(anticipation.start, scan);
|
|
127
|
+
if (occurrences % 2 === 0) {
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
} else {
|
|
131
|
+
const startIndex = lastStart.index + lastStart[0].length;
|
|
132
|
+
const hasEnd = Boolean(findMatchAfter(anticipation.end, scan, startIndex));
|
|
133
|
+
if (hasEnd) continue;
|
|
134
|
+
}
|
|
135
|
+
const appendValue = typeof anticipation.append === "function" ? anticipation.append(lastStart, content) : anticipation.append;
|
|
136
|
+
if (!appendValue) continue;
|
|
137
|
+
return appendValue;
|
|
138
|
+
}
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
75
141
|
/**
|
|
76
142
|
* Clear the memoization cache
|
|
77
143
|
*/
|
|
@@ -96,10 +162,27 @@ var InlineParser = class {
|
|
|
96
162
|
* Register default plugins with proper precedence ordering
|
|
97
163
|
* Lower priority numbers = higher precedence (run first)
|
|
98
164
|
*/
|
|
99
|
-
registerDefaultPlugins() {
|
|
165
|
+
registerDefaultPlugins(options) {
|
|
166
|
+
if (options.enableMath) {
|
|
167
|
+
this.registerPlugin({
|
|
168
|
+
id: "math-display",
|
|
169
|
+
priority: 0,
|
|
170
|
+
re: /\$\$([\s\S]+?)\$\$/g,
|
|
171
|
+
toNode: (match) => ({ kind: "math-display", tex: match[1].trim() }),
|
|
172
|
+
fastCheck: (text) => text.indexOf("$$") !== -1
|
|
173
|
+
});
|
|
174
|
+
this.registerPlugin({
|
|
175
|
+
id: "math-inline",
|
|
176
|
+
priority: 1,
|
|
177
|
+
re: /\$([^$\n]+?)\$/g,
|
|
178
|
+
// Non-greedy to prevent spanning multiple expressions
|
|
179
|
+
toNode: (match) => ({ kind: "math-inline", tex: match[1].trim() }),
|
|
180
|
+
fastCheck: (text) => text.indexOf("$") !== -1
|
|
181
|
+
});
|
|
182
|
+
}
|
|
100
183
|
this.registerPlugin({
|
|
101
184
|
id: "escaped-character",
|
|
102
|
-
priority:
|
|
185
|
+
priority: 2,
|
|
103
186
|
re: /\\([\\`*_{}\[\]()#+\-.!>])/g,
|
|
104
187
|
toNode: (match) => ({
|
|
105
188
|
kind: "text",
|
|
@@ -108,19 +191,11 @@ var InlineParser = class {
|
|
|
108
191
|
fastCheck: (text) => text.indexOf("\\") !== -1
|
|
109
192
|
});
|
|
110
193
|
this.registerPlugin({
|
|
111
|
-
id: "
|
|
112
|
-
priority: 1,
|
|
113
|
-
re: /\$\$([^$]+?)\$\$/g,
|
|
114
|
-
toNode: (match) => ({ kind: "math-display", tex: match[1].trim() }),
|
|
115
|
-
fastCheck: (text) => text.indexOf("$$") !== -1
|
|
116
|
-
});
|
|
117
|
-
this.registerPlugin({
|
|
118
|
-
id: "math-inline",
|
|
194
|
+
id: "hard-break",
|
|
119
195
|
priority: 2,
|
|
120
|
-
re:
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
fastCheck: (text) => text.indexOf("$") !== -1
|
|
196
|
+
re: /\\\r?\n| {2,}\r?\n/g,
|
|
197
|
+
toNode: (_match) => ({ kind: "br" }),
|
|
198
|
+
fastCheck: (text) => text.indexOf("\n") !== -1 || text.indexOf("\r") !== -1
|
|
124
199
|
});
|
|
125
200
|
this.registerPlugin({
|
|
126
201
|
id: "code-spans",
|
|
@@ -191,9 +266,16 @@ var InlineParser = class {
|
|
|
191
266
|
this.registerPlugin({
|
|
192
267
|
id: "citations",
|
|
193
268
|
priority: 10,
|
|
194
|
-
re: /\[\^([^\]]+)\]|@cite\{([^}]+)\}/g,
|
|
195
|
-
toNode: (match) => ({ kind: "citation", id: match[1] || match[2] }),
|
|
196
|
-
fastCheck: (text) => text.indexOf("@") !== -1 || text.indexOf("[^") !== -1
|
|
269
|
+
re: /\[\^([^\]\n]+)\]|@cite\{([^}\n]+)\}|\{cite:([^}\n]+)\}/g,
|
|
270
|
+
toNode: (match) => ({ kind: "citation", id: match[1] || match[2] || match[3] }),
|
|
271
|
+
fastCheck: (text) => text.indexOf("@cite") !== -1 || text.indexOf("[^") !== -1 || text.indexOf("{cite:") !== -1,
|
|
272
|
+
anticipation: {
|
|
273
|
+
start: /@cite\{|\{cite:/g,
|
|
274
|
+
end: /\}/g,
|
|
275
|
+
full: /@cite\{[^}\n]+?\}|\{cite:[^}\n]+?\}/g,
|
|
276
|
+
append: "}",
|
|
277
|
+
maxScanChars: 120
|
|
278
|
+
}
|
|
197
279
|
});
|
|
198
280
|
this.registerPlugin({
|
|
199
281
|
id: "mentions",
|
package/dist/inline-parser.d.cts
CHANGED
|
@@ -7,6 +7,11 @@ interface InlineParserOptions {
|
|
|
7
7
|
* parsing many intermediate states.
|
|
8
8
|
*/
|
|
9
9
|
maxCacheEntries?: number;
|
|
10
|
+
/**
|
|
11
|
+
* Enable parsing `$...$` and `$$...$$` math nodes.
|
|
12
|
+
* Defaults to `true`.
|
|
13
|
+
*/
|
|
14
|
+
enableMath?: boolean;
|
|
10
15
|
}
|
|
11
16
|
interface InlineParseOptions {
|
|
12
17
|
/**
|
|
@@ -31,6 +36,11 @@ declare class InlineParser {
|
|
|
31
36
|
* Parse inline content with memoization
|
|
32
37
|
*/
|
|
33
38
|
parse(content: string, options?: InlineParseOptions): InlineNode[];
|
|
39
|
+
/**
|
|
40
|
+
* Streaming regex anticipation helper. Returns an append string if a plugin
|
|
41
|
+
* declares an incomplete match at the end of the buffer.
|
|
42
|
+
*/
|
|
43
|
+
getRegexAnticipationAppend(content: string): string | null;
|
|
34
44
|
/**
|
|
35
45
|
* Clear the memoization cache
|
|
36
46
|
*/
|