@fuzdev/mdz 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Mdz.svelte +15 -7
- package/dist/Mdz.svelte.d.ts +9 -2
- package/dist/Mdz.svelte.d.ts.map +1 -1
- package/dist/MdzNodeView.svelte +118 -112
- package/dist/MdzNodeView.svelte.d.ts +6 -2
- package/dist/MdzNodeView.svelte.d.ts.map +1 -1
- package/dist/MdzStream.svelte +3 -3
- package/dist/MdzStream.svelte.d.ts.map +1 -1
- package/dist/MdzStreamNodeView.svelte +117 -114
- package/dist/MdzStreamNodeView.svelte.d.ts +6 -2
- package/dist/MdzStreamNodeView.svelte.d.ts.map +1 -1
- package/dist/mdz.d.ts +6 -0
- package/dist/mdz.d.ts.map +1 -1
- package/dist/mdz.js +8 -0
- package/dist/mdz_debug_work.d.ts +38 -0
- package/dist/mdz_debug_work.d.ts.map +1 -0
- package/dist/mdz_debug_work.js +37 -0
- package/dist/mdz_helpers.d.ts +61 -0
- package/dist/mdz_helpers.d.ts.map +1 -1
- package/dist/mdz_helpers.js +72 -0
- package/dist/mdz_lexer.d.ts +12 -0
- package/dist/mdz_lexer.d.ts.map +1 -1
- package/dist/mdz_lexer.js +164 -24
- package/dist/mdz_stream_parser_inline.d.ts.map +1 -1
- package/dist/mdz_stream_parser_inline.js +19 -17
- package/dist/mdz_stream_parser_link.d.ts.map +1 -1
- package/dist/mdz_stream_parser_link.js +88 -56
- package/dist/mdz_stream_parser_list.d.ts.map +1 -1
- package/dist/mdz_stream_parser_list.js +10 -2
- package/dist/mdz_stream_parser_state.d.ts +29 -0
- package/dist/mdz_stream_parser_state.d.ts.map +1 -1
- package/dist/mdz_stream_parser_state.js +52 -3
- package/dist/mdz_stream_parser_table.d.ts +3 -2
- package/dist/mdz_stream_parser_table.d.ts.map +1 -1
- package/dist/mdz_stream_parser_table.js +18 -7
- package/dist/mdz_stream_parser_text.d.ts.map +1 -1
- package/dist/mdz_stream_parser_text.js +35 -24
- package/dist/mdz_to_svelte.d.ts.map +1 -1
- package/dist/mdz_to_svelte.js +28 -31
- package/dist/mdz_token_parser.d.ts.map +1 -1
- package/dist/mdz_token_parser.js +15 -16
- package/dist/svelte_preprocess_mdz.js +31 -37
- package/package.json +7 -4
- package/src/lib/mdz.ts +9 -0
- package/src/lib/mdz_debug_work.ts +38 -0
- package/src/lib/mdz_helpers.ts +97 -0
- package/src/lib/mdz_lexer.ts +168 -28
- package/src/lib/mdz_stream_parser_inline.ts +21 -17
- package/src/lib/mdz_stream_parser_link.ts +89 -56
- package/src/lib/mdz_stream_parser_list.ts +10 -4
- package/src/lib/mdz_stream_parser_state.ts +76 -4
- package/src/lib/mdz_stream_parser_table.ts +18 -7
- package/src/lib/mdz_stream_parser_text.ts +38 -37
- package/src/lib/mdz_to_svelte.ts +23 -31
- package/src/lib/mdz_token_parser.ts +17 -20
- package/src/lib/svelte_preprocess_mdz.ts +34 -35
package/src/lib/mdz.ts
CHANGED
|
@@ -89,6 +89,15 @@ export class MdzTableCellParser {
|
|
|
89
89
|
this.#lexer = new MdzLexer(text);
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
+
/**
|
|
93
|
+
* Rebind to a new buffer so one instance serves a whole table — the
|
|
94
|
+
* streaming path rebinds per row (the buffer string changes across feeds;
|
|
95
|
+
* the lexer's search memo survives when it doesn't).
|
|
96
|
+
*/
|
|
97
|
+
rebind(text: string): void {
|
|
98
|
+
this.#lexer.rebind(text);
|
|
99
|
+
}
|
|
100
|
+
|
|
92
101
|
parse(cell_start: number, cell_end: number): Array<MdzNode> {
|
|
93
102
|
this.#parser.reset(this.#lexer.lex_table_cell(cell_start, cell_end));
|
|
94
103
|
return this.#parser.parse_inline_run();
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic scan-work counter for the complexity-ratio guards
|
|
3
|
+
* (`mdz_scaling_ratio.test.ts`). Off by default; when `enabled`, the parser hot
|
|
4
|
+
* paths accumulate the number of characters scanned (and tag-stack frames
|
|
5
|
+
* walked) into `total`. Unlike wall-clock timing this is a pure function of
|
|
6
|
+
* input + algorithm — invariant across machines, runs, and thermal state — so a
|
|
7
|
+
* super-linear regression shows up as a work ratio > ~linear regardless of the
|
|
8
|
+
* ~2× machine variance that makes absolute timing unusable here. Its sole
|
|
9
|
+
* consumer is `mdz_scaling_ratio.test.ts`.
|
|
10
|
+
*
|
|
11
|
+
* Coverage is the dominant re-scan choke points — `#index_of` + `#tokenize_text`
|
|
12
|
+
* (sync) and the streaming memo scans, `consume_text_run`, and `try_close_tag`'s
|
|
13
|
+
* stack walk — not every scan. A super-linear regression isolated to an
|
|
14
|
+
* *uncounted* probe (e.g. `#has_paragraph_break_between`, the tag-name collection
|
|
15
|
+
* loop, the URL/path probes) wouldn't move the ratio; extend the instrumentation
|
|
16
|
+
* if hot cost ever migrates there.
|
|
17
|
+
*
|
|
18
|
+
* Zero store when disabled: each instrumented site guards on `enabled` (a single
|
|
19
|
+
* branch-predicted property load, the same category as the shipped `DEV`
|
|
20
|
+
* checks). It is deliberately NOT the `esm-env` `DEV` flag — `DEV` is true under
|
|
21
|
+
* every `gro test`/`gro dev` run, which would leave the counter live in ordinary
|
|
22
|
+
* development; this flag is flipped true only by the ratio test.
|
|
23
|
+
*
|
|
24
|
+
* @nodocs
|
|
25
|
+
*/
|
|
26
|
+
export const mdz_debug_work: {enabled: boolean; total: number} = {enabled: false, total: 0};
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Reset the counter and set whether the hot paths accumulate into it. Tests call
|
|
30
|
+
* `mdz_debug_work_reset()` immediately before a parse and read
|
|
31
|
+
* `mdz_debug_work.total` immediately after; pass `false` to disable and zero it.
|
|
32
|
+
*
|
|
33
|
+
* @nodocs
|
|
34
|
+
*/
|
|
35
|
+
export const mdz_debug_work_reset = (enabled = true): void => {
|
|
36
|
+
mdz_debug_work.enabled = enabled;
|
|
37
|
+
mdz_debug_work.total = 0;
|
|
38
|
+
};
|
package/src/lib/mdz_helpers.ts
CHANGED
|
@@ -99,6 +99,23 @@ export const MIN_CODEBLOCK_BACKTICKS = 3; // Code blocks require minimum 3 backt
|
|
|
99
99
|
export const MAX_LIST_NUMBER_DIGITS = 9; // Ordered list markers cap at 9 digits (CommonMark's cap)
|
|
100
100
|
/** @nodocs */
|
|
101
101
|
export const MAX_HEADING_LEVEL = 6; // Headings support levels 1-6
|
|
102
|
+
/**
|
|
103
|
+
* Cap on inline link/tag nesting depth. Reached at this many open
|
|
104
|
+
* `Link`/`Element`/`Component` containers, a further `[` or `<tag>` opener
|
|
105
|
+
* renders as literal text in both parsers rather than opening — a strict
|
|
106
|
+
* untrusted-content dialect's bound on pathological nesting (CommonMark
|
|
107
|
+
* permits such a limit). It bounds both the sync lexer's recursion depth (no
|
|
108
|
+
* stack overflow) and, with the failure memo, its revert re-scan work (linear
|
|
109
|
+
* instead of quadratic) on adversarial nested `[`/`<`. Deliberately far above
|
|
110
|
+
* any real content (inline nesting rarely exceeds a handful); only pathological
|
|
111
|
+
* input reaches it. Mirrored in the streaming parser so the two agree on where
|
|
112
|
+
* the cap bites. Delimiters (`**`/`_`/`~~`/`` ` ``) are uncounted: first-closer-wins
|
|
113
|
+
* keeps them from nesting deeply on their own, and mixed delimiter/link chains
|
|
114
|
+
* only deep-nest through the links, which this bounds.
|
|
115
|
+
*
|
|
116
|
+
* @nodocs
|
|
117
|
+
*/
|
|
118
|
+
export const MAX_INLINE_NESTING_DEPTH = 100;
|
|
102
119
|
/** @nodocs */
|
|
103
120
|
export const HTTPS_PREFIX_LENGTH = 8; // Length of "https://"
|
|
104
121
|
/** @nodocs */
|
|
@@ -547,6 +564,40 @@ const PATH_CHAR_TABLE: Uint8Array = (() => {
|
|
|
547
564
|
export const is_valid_path_char = (char_code: number): boolean =>
|
|
548
565
|
char_code < 128 && PATH_CHAR_TABLE[char_code] === 1;
|
|
549
566
|
|
|
567
|
+
// Character classes for the plain-text run scanners (the hottest loops in
|
|
568
|
+
// both parsers) — one table load per character instead of a comparison chain.
|
|
569
|
+
// A zero class is the common case: the char can't end or interrupt a text run.
|
|
570
|
+
|
|
571
|
+
/** Always ends a text run: `` ` `` `*` `_` `~` `[` `]` `<`. @nodocs */
|
|
572
|
+
export const TEXT_CLASS_STOP = 1;
|
|
573
|
+
/** `\n` — run handling is context-dependent (soft break, paragraph break, block lookahead). @nodocs */
|
|
574
|
+
export const TEXT_CLASS_NEWLINE = 2;
|
|
575
|
+
/** `h`/`H` — potential URL scheme start, needs the prefix probe. @nodocs */
|
|
576
|
+
export const TEXT_CLASS_URL = 4;
|
|
577
|
+
/** `/` and `.` — potential path start, needs the boundary probe. @nodocs */
|
|
578
|
+
export const TEXT_CLASS_PATH = 8;
|
|
579
|
+
/** `\` — a literal-pipe escape candidate inside table cells only. @nodocs */
|
|
580
|
+
export const TEXT_CLASS_BACKSLASH = 16;
|
|
581
|
+
|
|
582
|
+
/** @nodocs */
|
|
583
|
+
export const TEXT_SCAN_CLASS: Uint8Array = (() => {
|
|
584
|
+
const t = new Uint8Array(128);
|
|
585
|
+
t[BACKTICK] = TEXT_CLASS_STOP;
|
|
586
|
+
t[ASTERISK] = TEXT_CLASS_STOP;
|
|
587
|
+
t[UNDERSCORE] = TEXT_CLASS_STOP;
|
|
588
|
+
t[TILDE] = TEXT_CLASS_STOP;
|
|
589
|
+
t[LEFT_BRACKET] = TEXT_CLASS_STOP;
|
|
590
|
+
t[RIGHT_BRACKET] = TEXT_CLASS_STOP;
|
|
591
|
+
t[LEFT_ANGLE] = TEXT_CLASS_STOP;
|
|
592
|
+
t[NEWLINE] = TEXT_CLASS_NEWLINE;
|
|
593
|
+
t[H_LOWER] = TEXT_CLASS_URL;
|
|
594
|
+
t[H_UPPER] = TEXT_CLASS_URL;
|
|
595
|
+
t[SLASH] = TEXT_CLASS_PATH;
|
|
596
|
+
t[PERIOD] = TEXT_CLASS_PATH;
|
|
597
|
+
t[BACKSLASH] = TEXT_CLASS_BACKSLASH;
|
|
598
|
+
return t;
|
|
599
|
+
})();
|
|
600
|
+
|
|
550
601
|
/**
|
|
551
602
|
* Trim trailing punctuation from URL/path per RFC 3986 and GFM rules.
|
|
552
603
|
* - Trims simple trailing: .,;:!?]
|
|
@@ -760,6 +811,52 @@ export const mdz_resolve_relative_path = (reference: string, base: string): stri
|
|
|
760
811
|
return segments.join('/');
|
|
761
812
|
};
|
|
762
813
|
|
|
814
|
+
/**
|
|
815
|
+
* How a `Link` node's `reference` should render, shared by all three renderers
|
|
816
|
+
* (`MdzNodeView`, `MdzStreamNodeView`, `mdz_to_svelte`) so the safety gate and
|
|
817
|
+
* the resolve-vs-raw classification can't drift between them.
|
|
818
|
+
* @nodocs
|
|
819
|
+
*/
|
|
820
|
+
export type MdzLinkRender =
|
|
821
|
+
/** Unsafe protocol — render children only, no `<a>`. */
|
|
822
|
+
| {kind: 'unsafe'}
|
|
823
|
+
/** Route/relative path — wrap in `resolve()` (needs `$app/paths`). */
|
|
824
|
+
| {kind: 'resolve'; href: string}
|
|
825
|
+
/** Internal fragment/query/relative/bare ref — raw `href`, no `resolve()`. */
|
|
826
|
+
| {kind: 'internal'; href: string}
|
|
827
|
+
/** External link — raw `href` plus `target="_blank" rel="noopener"`. */
|
|
828
|
+
| {kind: 'external'; href: string};
|
|
829
|
+
|
|
830
|
+
/**
|
|
831
|
+
* Classify a `Link` reference into how it should render. Pure — no rendering,
|
|
832
|
+
* no context; `base` is the resolved base path (or `undefined`).
|
|
833
|
+
* @nodocs
|
|
834
|
+
*/
|
|
835
|
+
export const mdz_classify_link = (
|
|
836
|
+
reference: string,
|
|
837
|
+
link_type: 'internal' | 'external' | undefined,
|
|
838
|
+
base: string | undefined,
|
|
839
|
+
): MdzLinkRender => {
|
|
840
|
+
if (!mdz_is_safe_reference(reference)) return {kind: 'unsafe'};
|
|
841
|
+
if (link_type === 'internal') {
|
|
842
|
+
if (reference.startsWith('.') && base) {
|
|
843
|
+
return {kind: 'resolve', href: mdz_resolve_relative_path(reference, base)};
|
|
844
|
+
}
|
|
845
|
+
// fragment/query/relative/bare — `resolve()` only accepts absolute paths or
|
|
846
|
+
// route ids and throws on anything else
|
|
847
|
+
if (
|
|
848
|
+
reference.startsWith('#') ||
|
|
849
|
+
reference.startsWith('?') ||
|
|
850
|
+
reference.startsWith('.') ||
|
|
851
|
+
!reference.startsWith('/')
|
|
852
|
+
) {
|
|
853
|
+
return {kind: 'internal', href: reference};
|
|
854
|
+
}
|
|
855
|
+
return {kind: 'resolve', href: reference};
|
|
856
|
+
}
|
|
857
|
+
return {kind: 'external', href: reference};
|
|
858
|
+
};
|
|
859
|
+
|
|
763
860
|
/**
|
|
764
861
|
* Push a node into a children array, coalescing with the previous Text node.
|
|
765
862
|
* Mutates `prev.content` and `prev.end` when both are Text, avoiding array growth
|
package/src/lib/mdz_lexer.ts
CHANGED
|
@@ -38,6 +38,7 @@ import {
|
|
|
38
38
|
HR_HYPHEN_COUNT,
|
|
39
39
|
MIN_CODEBLOCK_BACKTICKS,
|
|
40
40
|
MAX_HEADING_LEVEL,
|
|
41
|
+
MAX_INLINE_NESTING_DEPTH,
|
|
41
42
|
MAX_LIST_NUMBER_DIGITS,
|
|
42
43
|
match_url_prefix_case_insensitive,
|
|
43
44
|
is_at_absolute_path,
|
|
@@ -50,10 +51,15 @@ import {
|
|
|
50
51
|
mdz_split_table_row,
|
|
51
52
|
mdz_parse_table_delimiter,
|
|
52
53
|
PIPE,
|
|
53
|
-
|
|
54
|
+
TEXT_CLASS_BACKSLASH,
|
|
55
|
+
TEXT_CLASS_NEWLINE,
|
|
56
|
+
TEXT_CLASS_STOP,
|
|
57
|
+
TEXT_CLASS_URL,
|
|
58
|
+
TEXT_SCAN_CLASS,
|
|
54
59
|
} from './mdz_helpers.ts';
|
|
55
60
|
|
|
56
61
|
import type {MdzTableAlign} from './mdz.ts';
|
|
62
|
+
import {mdz_debug_work} from './mdz_debug_work.ts';
|
|
57
63
|
|
|
58
64
|
//
|
|
59
65
|
// Token types
|
|
@@ -304,11 +310,69 @@ export class MdzLexer {
|
|
|
304
310
|
* and `\|` unescapes to a literal pipe.
|
|
305
311
|
*/
|
|
306
312
|
#in_table_cell: boolean = false;
|
|
313
|
+
/**
|
|
314
|
+
* Start offsets of markdown-link opens (`[`) that definitively failed — so
|
|
315
|
+
* a re-lex from before the `[` short-circuits to literal text instead of
|
|
316
|
+
* re-attempting (and re-descending into) the same failing link. Without it,
|
|
317
|
+
* the revert-and-re-tokenize failure path is exponential on nested `[`
|
|
318
|
+
* (a ~30-byte `[`×n input hangs). A link's outcome (does a valid `](ref)`
|
|
319
|
+
* follow?) is independent of `#max_search_index`, so `start` alone keys it.
|
|
320
|
+
* The outcome is a pure function of the immutable text, so the memo is valid
|
|
321
|
+
* for the whole lex and across table-cell ranges; cleared on `rebind`.
|
|
322
|
+
*/
|
|
323
|
+
#failed_link_starts: Set<number> = new Set();
|
|
324
|
+
/**
|
|
325
|
+
* The tag analogue of `#failed_link_starts`, keyed `${start}:${bound}`. A
|
|
326
|
+
* tag's closer search respects `#max_search_index`, so the same `start` can
|
|
327
|
+
* fail under a narrow bound yet succeed under the wider one a re-lex uses —
|
|
328
|
+
* the bound is part of the key.
|
|
329
|
+
*/
|
|
330
|
+
#failed_tag: Set<string> = new Set();
|
|
331
|
+
/**
|
|
332
|
+
* Current link/tag nesting depth — the number of open
|
|
333
|
+
* `link_text_open`/`tag_open` containers being recursed into. Bumped around
|
|
334
|
+
* `#tokenize_markdown_link`/`#tokenize_tag`'s children loops (not by
|
|
335
|
+
* delimiters, which can't nest deeply — see `MAX_INLINE_NESTING_DEPTH`). At
|
|
336
|
+
* the cap a further `[`/`<` renders literal instead of opening, bounding
|
|
337
|
+
* recursion depth (no stack overflow) and, with the failure memos, the
|
|
338
|
+
* revert re-scan (linear, not quadratic) on adversarial nesting.
|
|
339
|
+
*/
|
|
340
|
+
#inline_depth: number = 0;
|
|
341
|
+
/**
|
|
342
|
+
* Start offsets of `[`/`<` openers suppressed by the nesting-depth cap
|
|
343
|
+
* (`#inline_depth >= MAX_INLINE_NESTING_DEPTH`) — position-keyed like
|
|
344
|
+
* `#failed_link_starts` so a re-lex from a shallower depth short-circuits to
|
|
345
|
+
* literal text rather than re-opening the construct (which would reintroduce
|
|
346
|
+
* the quadratic the cap removes). A suppression is bound-independent, so
|
|
347
|
+
* `start` alone keys it; cleared on `rebind`.
|
|
348
|
+
*/
|
|
349
|
+
#depth_capped: Set<number> = new Set();
|
|
307
350
|
|
|
308
351
|
constructor(text: string) {
|
|
309
352
|
this.#text = text;
|
|
310
353
|
}
|
|
311
354
|
|
|
355
|
+
/**
|
|
356
|
+
* Rebind to a new text, so one lexer instance can be reused across inputs —
|
|
357
|
+
* the streaming table path reuses one across a table's rows via
|
|
358
|
+
* `MdzTableCellParser` instead of allocating a lexer (and its memo `Map`)
|
|
359
|
+
* per row. The search memos survive when the text is identical (they're
|
|
360
|
+
* valid over the immutable text — the common case for rows parsed from one
|
|
361
|
+
* buffered chunk) and clear otherwise. Per-parse cursor state is reset by
|
|
362
|
+
* `lex_table_cell` itself.
|
|
363
|
+
*
|
|
364
|
+
* @nodocs
|
|
365
|
+
*/
|
|
366
|
+
rebind(text: string): void {
|
|
367
|
+
if (this.#text === text) return;
|
|
368
|
+
this.#text = text;
|
|
369
|
+
this.#search_memo.clear();
|
|
370
|
+
this.#break_memo = null;
|
|
371
|
+
this.#failed_link_starts.clear();
|
|
372
|
+
this.#failed_tag.clear();
|
|
373
|
+
this.#depth_capped.clear();
|
|
374
|
+
}
|
|
375
|
+
|
|
312
376
|
/**
|
|
313
377
|
* Memoized `String#indexOf` over the immutable text.
|
|
314
378
|
*
|
|
@@ -325,6 +389,10 @@ export class MdzLexer {
|
|
|
325
389
|
return memo.result;
|
|
326
390
|
}
|
|
327
391
|
const result = this.#text.indexOf(needle, from);
|
|
392
|
+
if (mdz_debug_work.enabled) {
|
|
393
|
+
// scan span: `indexOf` examined [from, result] (found) or [from, EOF)
|
|
394
|
+
mdz_debug_work.total += (result === -1 ? this.#text.length : result) - from;
|
|
395
|
+
}
|
|
328
396
|
if (memo === undefined) {
|
|
329
397
|
this.#search_memo.set(needle, {from, result});
|
|
330
398
|
} else {
|
|
@@ -376,6 +444,9 @@ export class MdzLexer {
|
|
|
376
444
|
*/
|
|
377
445
|
lex_table_cell(cell_start: number, cell_end: number): Array<MdzToken> {
|
|
378
446
|
this.#tokens = [];
|
|
447
|
+
// depth is balanced back to 0 per cell, but pin it so a future early-return
|
|
448
|
+
// between an `#inline_depth++` and its decrement can't leak into the next cell
|
|
449
|
+
this.#inline_depth = 0;
|
|
379
450
|
this.#tokenize_cell_inline(cell_start, cell_end);
|
|
380
451
|
return this.#tokens;
|
|
381
452
|
}
|
|
@@ -1541,6 +1612,23 @@ export class MdzLexer {
|
|
|
1541
1612
|
|
|
1542
1613
|
#tokenize_markdown_link(): void {
|
|
1543
1614
|
const start = this.#index;
|
|
1615
|
+
// a link open at this offset already failed, or was suppressed by the
|
|
1616
|
+
// nesting cap — emit `[` as text instead of re-descending into the same
|
|
1617
|
+
// failing children (exponential on nested `[`); the short-circuit
|
|
1618
|
+
// reproduces the revert's exact net effect (`[` text, cursor at start + 1)
|
|
1619
|
+
if (this.#failed_link_starts.has(start) || this.#depth_capped.has(start)) {
|
|
1620
|
+
this.#index = start + 1;
|
|
1621
|
+
this.#emit_text('[', start);
|
|
1622
|
+
return;
|
|
1623
|
+
}
|
|
1624
|
+
// nesting-depth cap: too deep to open — render literal and record the
|
|
1625
|
+
// suppression so a shallower re-lex short-circuits here too
|
|
1626
|
+
if (this.#inline_depth >= MAX_INLINE_NESTING_DEPTH) {
|
|
1627
|
+
this.#depth_capped.add(start);
|
|
1628
|
+
this.#index = start + 1;
|
|
1629
|
+
this.#emit_text('[', start);
|
|
1630
|
+
return;
|
|
1631
|
+
}
|
|
1544
1632
|
this.#index++; // consume [ (dispatch guarantees it)
|
|
1545
1633
|
|
|
1546
1634
|
// Emit link_text_open
|
|
@@ -1549,11 +1637,13 @@ export class MdzLexer {
|
|
|
1549
1637
|
// Tokenize children until ] — only `]` delimits link text; a bare `)`
|
|
1550
1638
|
// is ordinary content (matching the streaming parser, which has no `)`
|
|
1551
1639
|
// dispatch case — the reference scan finds its own `)` after `](`)
|
|
1640
|
+
this.#inline_depth++;
|
|
1552
1641
|
while (this.#index < this.#text.length) {
|
|
1553
1642
|
if (this.#text.charCodeAt(this.#index) === RIGHT_BRACKET) break;
|
|
1554
1643
|
if (this.#is_at_paragraph_break()) break;
|
|
1555
1644
|
this.#tokenize_inline();
|
|
1556
1645
|
}
|
|
1646
|
+
this.#inline_depth--;
|
|
1557
1647
|
|
|
1558
1648
|
// Check for ]
|
|
1559
1649
|
if (this.#index >= this.#text.length || this.#text.charCodeAt(this.#index) !== RIGHT_BRACKET) {
|
|
@@ -1622,6 +1712,13 @@ export class MdzLexer {
|
|
|
1622
1712
|
}
|
|
1623
1713
|
|
|
1624
1714
|
#revert_tokens_from_link_open(start: number): void {
|
|
1715
|
+
// record the failure so a re-lex from before this `[` short-circuits.
|
|
1716
|
+
// Keyed by `start` alone — unlike `#failed_tag`'s `start:bound` — because a
|
|
1717
|
+
// link's outcome is bound-independent: the `]`-terminator scan (the children
|
|
1718
|
+
// loop) runs to `text.length` and never consults `#max_search_index`, so a
|
|
1719
|
+
// wider-bound re-lex can't move it. The bound reaches nested tags inside the
|
|
1720
|
+
// link text, but re-attempting those wider never flips the link's net result.
|
|
1721
|
+
this.#failed_link_starts.add(start);
|
|
1625
1722
|
// Find and remove link_text_open and all tokens after it
|
|
1626
1723
|
let open_idx = -1;
|
|
1627
1724
|
for (let i = this.#tokens.length - 1; i >= 0; i--) {
|
|
@@ -1639,6 +1736,32 @@ export class MdzLexer {
|
|
|
1639
1736
|
|
|
1640
1737
|
#tokenize_tag(): void {
|
|
1641
1738
|
const start = this.#index;
|
|
1739
|
+
// short-circuit a re-attempt of a tag open that already failed at this
|
|
1740
|
+
// offset+bound — re-descending into the same failing children is
|
|
1741
|
+
// exponential on nested same-name tags. Keyed by
|
|
1742
|
+
// `#max_search_index` (captured as `bound`) because the closer search is
|
|
1743
|
+
// bound-sensitive, so a wider-bound re-lex can legitimately succeed. The
|
|
1744
|
+
// key string is built only when the memo is non-empty (adversarial
|
|
1745
|
+
// input); prose never allocates it. The short-circuit reproduces a
|
|
1746
|
+
// revert's exact net effect (`<` text, cursor at start + 1).
|
|
1747
|
+
const bound = this.#max_search_index;
|
|
1748
|
+
if (
|
|
1749
|
+
this.#depth_capped.has(start) ||
|
|
1750
|
+
(this.#failed_tag.size !== 0 && this.#failed_tag.has(`${start}:${bound}`))
|
|
1751
|
+
) {
|
|
1752
|
+
this.#index = start + 1;
|
|
1753
|
+
this.#emit_text('<', start);
|
|
1754
|
+
return;
|
|
1755
|
+
}
|
|
1756
|
+
// nesting-depth cap: too deep to open a tag — render literal and record
|
|
1757
|
+
// the suppression (position-keyed) so a shallower re-lex short-circuits
|
|
1758
|
+
// here too, keeping the revert re-scan linear
|
|
1759
|
+
if (this.#inline_depth >= MAX_INLINE_NESTING_DEPTH) {
|
|
1760
|
+
this.#depth_capped.add(start);
|
|
1761
|
+
this.#index = start + 1;
|
|
1762
|
+
this.#emit_text('<', start);
|
|
1763
|
+
return;
|
|
1764
|
+
}
|
|
1642
1765
|
this.#index++; // consume <
|
|
1643
1766
|
|
|
1644
1767
|
// Tag name must start with a letter
|
|
@@ -1685,6 +1808,7 @@ export class MdzLexer {
|
|
|
1685
1808
|
|
|
1686
1809
|
// Check for >
|
|
1687
1810
|
if (this.#index >= this.#text.length || this.#text.charCodeAt(this.#index) !== RIGHT_ANGLE) {
|
|
1811
|
+
this.#failed_tag.add(`${start}:${bound}`);
|
|
1688
1812
|
this.#index = start + 1;
|
|
1689
1813
|
this.#emit_text('<', start);
|
|
1690
1814
|
return;
|
|
@@ -1703,11 +1827,13 @@ export class MdzLexer {
|
|
|
1703
1827
|
const search_limit = Math.min(this.#max_search_index, this.#text.length);
|
|
1704
1828
|
const closing_tag_pos = this.#index_of(closing_tag, this.#index);
|
|
1705
1829
|
if (closing_tag_pos === -1 || closing_tag_pos > search_limit) {
|
|
1830
|
+
this.#failed_tag.add(`${start}:${bound}`);
|
|
1706
1831
|
this.#index = start + 1;
|
|
1707
1832
|
this.#emit_text('<', start);
|
|
1708
1833
|
return;
|
|
1709
1834
|
}
|
|
1710
1835
|
if (this.#has_paragraph_break_between(this.#index, closing_tag_pos)) {
|
|
1836
|
+
this.#failed_tag.add(`${start}:${bound}`);
|
|
1711
1837
|
this.#index = start + 1;
|
|
1712
1838
|
this.#emit_text('<', start);
|
|
1713
1839
|
return;
|
|
@@ -1723,8 +1849,10 @@ export class MdzLexer {
|
|
|
1723
1849
|
// the code span). Recomputed per iteration because a nested same-name
|
|
1724
1850
|
// tag legitimately consumes the nearest closer (`<b>x<b>y</b></b>`).
|
|
1725
1851
|
const saved_max = this.#max_search_index;
|
|
1852
|
+
this.#inline_depth++;
|
|
1726
1853
|
while (this.#index < this.#text.length) {
|
|
1727
1854
|
if (this.#match(closing_tag)) {
|
|
1855
|
+
this.#inline_depth--;
|
|
1728
1856
|
this.#max_search_index = saved_max;
|
|
1729
1857
|
const close_start = this.#index;
|
|
1730
1858
|
this.#index += closing_tag.length;
|
|
@@ -1747,12 +1875,14 @@ export class MdzLexer {
|
|
|
1747
1875
|
this.#max_search_index = next_close;
|
|
1748
1876
|
this.#tokenize_inline();
|
|
1749
1877
|
}
|
|
1878
|
+
this.#inline_depth--;
|
|
1750
1879
|
|
|
1751
1880
|
// No valid closer left — revert: drop the open token and all consumed
|
|
1752
1881
|
// children tokens, emit `<` as text, and re-lex from the next char
|
|
1753
1882
|
// (the `#revert_tokens_from_link_open` pattern; the old fallthrough
|
|
1754
1883
|
// left a dangling `tag_open` whose parser fallback discarded content)
|
|
1755
1884
|
this.#max_search_index = saved_max;
|
|
1885
|
+
this.#failed_tag.add(`${start}:${bound}`);
|
|
1756
1886
|
this.#tokens.splice(open_token_index);
|
|
1757
1887
|
this.#index = start + 1;
|
|
1758
1888
|
this.#emit_text('<', start);
|
|
@@ -1780,16 +1910,33 @@ export class MdzLexer {
|
|
|
1780
1910
|
let parts: Array<string> | null = null;
|
|
1781
1911
|
let part_start = start;
|
|
1782
1912
|
|
|
1783
|
-
|
|
1913
|
+
// in a table cell the trimmed cell end is a hard stop (no newline to
|
|
1914
|
+
// halt on mid-line) — hoisted to the loop bound
|
|
1915
|
+
const scan_end = this.#in_table_cell
|
|
1916
|
+
? Math.min(this.#max_search_index, this.#text.length)
|
|
1917
|
+
: this.#text.length;
|
|
1918
|
+
|
|
1919
|
+
while (this.#index < scan_end) {
|
|
1784
1920
|
const char_code = this.#text.charCodeAt(this.#index);
|
|
1921
|
+
// one table load classifies the char; zero (the common case — the
|
|
1922
|
+
// char can't end or interrupt the run) skips the dispatch below.
|
|
1923
|
+
// `TEXT_SCAN_CLASS` is shared with the streaming scanner
|
|
1924
|
+
// (`consume_text_run` in `mdz_stream_parser_text.ts`); the two dispatch
|
|
1925
|
+
// bodies intentionally differ around it — this one handles `\|` escapes
|
|
1926
|
+
// and list-run strip, that one URL speculation across chunk boundaries.
|
|
1927
|
+
const char_class = char_code < 128 ? TEXT_SCAN_CLASS[char_code]! : 0;
|
|
1928
|
+
if (char_class === 0) {
|
|
1929
|
+
this.#index++;
|
|
1930
|
+
continue;
|
|
1931
|
+
}
|
|
1932
|
+
|
|
1933
|
+
// Stop at special characters
|
|
1934
|
+
if ((char_class & TEXT_CLASS_STOP) !== 0) break;
|
|
1785
1935
|
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
// scoped here)
|
|
1789
|
-
if (this.#in_table_cell) {
|
|
1790
|
-
if (this.#index >= this.#max_search_index) break;
|
|
1936
|
+
if ((char_class & TEXT_CLASS_BACKSLASH) !== 0) {
|
|
1937
|
+
// `\|` is a literal pipe — the only mdz escape, scoped to table cells
|
|
1791
1938
|
if (
|
|
1792
|
-
|
|
1939
|
+
this.#in_table_cell &&
|
|
1793
1940
|
this.#index + 1 < this.#max_search_index &&
|
|
1794
1941
|
this.#text.charCodeAt(this.#index + 1) === PIPE
|
|
1795
1942
|
) {
|
|
@@ -1799,22 +1946,11 @@ export class MdzLexer {
|
|
|
1799
1946
|
part_start = this.#index;
|
|
1800
1947
|
continue;
|
|
1801
1948
|
}
|
|
1949
|
+
this.#index++;
|
|
1950
|
+
continue;
|
|
1802
1951
|
}
|
|
1803
1952
|
|
|
1804
|
-
|
|
1805
|
-
if (
|
|
1806
|
-
char_code === BACKTICK ||
|
|
1807
|
-
char_code === ASTERISK ||
|
|
1808
|
-
char_code === UNDERSCORE ||
|
|
1809
|
-
char_code === TILDE ||
|
|
1810
|
-
char_code === LEFT_BRACKET ||
|
|
1811
|
-
char_code === RIGHT_BRACKET ||
|
|
1812
|
-
char_code === LEFT_ANGLE
|
|
1813
|
-
) {
|
|
1814
|
-
break;
|
|
1815
|
-
}
|
|
1816
|
-
|
|
1817
|
-
if (char_code === NEWLINE) {
|
|
1953
|
+
if ((char_class & TEXT_CLASS_NEWLINE) !== 0) {
|
|
1818
1954
|
if (this.#list_run_strip) {
|
|
1819
1955
|
if (this.#index >= this.#max_search_index) break; // the run's terminator
|
|
1820
1956
|
// soft break: keep the newline, strip the next line's leading
|
|
@@ -1853,20 +1989,24 @@ export class MdzLexer {
|
|
|
1853
1989
|
break;
|
|
1854
1990
|
}
|
|
1855
1991
|
}
|
|
1992
|
+
this.#index++; // single newline — ordinary text
|
|
1993
|
+
continue;
|
|
1856
1994
|
}
|
|
1857
1995
|
|
|
1858
|
-
//
|
|
1859
|
-
if (
|
|
1860
|
-
(
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
) {
|
|
1996
|
+
// URL or internal path mid-text (class guard avoids probing every char)
|
|
1997
|
+
if ((char_class & TEXT_CLASS_URL) !== 0) {
|
|
1998
|
+
if (this.#is_at_url()) break;
|
|
1999
|
+
} else if (char_code === SLASH) {
|
|
2000
|
+
if (is_at_absolute_path(this.#text, this.#index)) break;
|
|
2001
|
+
} else if (is_at_relative_path(this.#text, this.#index)) {
|
|
1864
2002
|
break;
|
|
1865
2003
|
}
|
|
1866
2004
|
|
|
1867
2005
|
this.#index++;
|
|
1868
2006
|
}
|
|
1869
2007
|
|
|
2008
|
+
if (mdz_debug_work.enabled) mdz_debug_work.total += this.#index - start;
|
|
2009
|
+
|
|
1870
2010
|
// Ensure we always consume at least one character
|
|
1871
2011
|
if (this.#index === start && this.#index < this.#text.length) {
|
|
1872
2012
|
this.#index++;
|
|
@@ -339,17 +339,24 @@ export const scan_closer_boundary = (
|
|
|
339
339
|
/** @nodocs */
|
|
340
340
|
export const try_code = (state: MdzStreamParserState, forced = false): boolean => {
|
|
341
341
|
const start = state.pos;
|
|
342
|
-
|
|
342
|
+
const search_start = start + 1; // past opening backtick
|
|
343
343
|
|
|
344
344
|
// compute search boundary: don't scan past open formatting delimiters
|
|
345
|
-
const search_limit = code_search_limit(state,
|
|
345
|
+
const search_limit = code_search_limit(state, search_start);
|
|
346
346
|
|
|
347
|
-
//
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
347
|
+
// find the first closing backtick or newline (memoized — a candidate that
|
|
348
|
+
// holds under open formatting re-enters here on every feed, and a raw scan
|
|
349
|
+
// would re-cover the same growing region each time, going quadratic on a
|
|
350
|
+
// single long line)
|
|
351
|
+
const close_pos = buffer_index_of(state, '`', search_start);
|
|
352
|
+
const newline_pos = buffer_index_of(state, '\n', search_start);
|
|
353
|
+
const terminator =
|
|
354
|
+
close_pos !== -1 && (newline_pos === -1 || close_pos < newline_pos) ? close_pos : newline_pos;
|
|
355
|
+
|
|
356
|
+
if (terminator !== -1 && terminator < search_limit) {
|
|
357
|
+
if (terminator === close_pos) {
|
|
351
358
|
// found close
|
|
352
|
-
const content = state.buffer.slice(
|
|
359
|
+
const content = state.buffer.slice(search_start, terminator);
|
|
353
360
|
if (content.length === 0) {
|
|
354
361
|
// empty code — treat as text
|
|
355
362
|
consume_delimiter_as_text(state, '``');
|
|
@@ -364,20 +371,17 @@ export const try_code = (state: MdzStreamParserState, forced = false): boolean =
|
|
|
364
371
|
content,
|
|
365
372
|
text_type: 'Code',
|
|
366
373
|
start: offset(state, start),
|
|
367
|
-
end: offset(state,
|
|
374
|
+
end: offset(state, terminator + 1),
|
|
368
375
|
});
|
|
369
376
|
state.active_text_id = null;
|
|
370
|
-
state.pos =
|
|
371
|
-
state.column +=
|
|
377
|
+
state.pos = terminator + 1;
|
|
378
|
+
state.column += terminator + 1 - start;
|
|
372
379
|
state.prev_char = BACKTICK;
|
|
373
380
|
return true;
|
|
374
381
|
}
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
return true;
|
|
379
|
-
}
|
|
380
|
-
i++;
|
|
382
|
+
// newline before any close — inline code can't span lines
|
|
383
|
+
consume_delimiter_as_text(state, '`');
|
|
384
|
+
return true;
|
|
381
385
|
}
|
|
382
386
|
|
|
383
387
|
// hit search limit (open formatting delimiter boundary) — treat backtick as
|
|
@@ -386,7 +390,7 @@ export const try_code = (state: MdzStreamParserState, forced = false): boolean =
|
|
|
386
390
|
// processed), the one-shot parse rejects that italic and scans unbounded —
|
|
387
391
|
// see the `MdzStreamParser` TSDoc and the parity suite's expected-divergence
|
|
388
392
|
// battery
|
|
389
|
-
if (
|
|
393
|
+
if (search_limit < state.buffer.length) {
|
|
390
394
|
consume_delimiter_as_text(state, '`');
|
|
391
395
|
return true;
|
|
392
396
|
}
|