@fuzdev/mdz 0.1.0

package/dist/mdz_lexer.js

@@ -0,0 +1,1405 @@
+/**
+ * mdz lexer — tokenizes input into a flat `MdzToken[]` stream.
+ *
+ * Phase 1 of the synchronous parsing pipeline behind `mdz_parse`.
+ * Phase 2 (token parser) is in `mdz_token_parser.ts`.
+ *
+ * @module
+ */
+import { mdz_is_url, mdz_is_safe_reference, is_letter, is_tag_name_char, is_word_char, is_valid_path_char, mdz_trim_url_punctuation, BACKTICK, ASTERISK, UNDERSCORE, TILDE, NEWLINE, HYPHEN, HASH, SPACE, LEFT_ANGLE, RIGHT_ANGLE, SLASH, PERIOD, LEFT_BRACKET, LEFT_PAREN, RIGHT_BRACKET, A_UPPER, Z_UPPER, ZERO, ONE, NINE, HR_HYPHEN_COUNT, MIN_CODEBLOCK_BACKTICKS, MAX_HEADING_LEVEL, MAX_LIST_NUMBER_DIGITS, match_url_prefix_case_insensitive, is_at_absolute_path, is_at_relative_path, is_line_whitespace, mdz_match_blockquote_prefix, mdz_blockquote_line_has_content, mdz_blockquote_strip_width, mdz_remap_segments, } from './mdz_helpers.js';
+//
+// Lexer
+//
+/**
+ * Tokenizes mdz input into a flat `MdzToken` stream for `MdzTokenParser`.
+ *
+ * The lexer owns all structural decisions — block boundaries, list level
+ * stacks, blockquote prefix stripping (sub-lexing quote content as a
+ * mini-document with positions remapped to source offsets), and inline
+ * delimiter pairing bounded by `#max_search_index` — so the token parser
+ * only assembles the tree.
+ */
+export class MdzLexer {
+    #text;
+    #index = 0;
+    #tokens = [];
+    #max_search_index = Number.MAX_SAFE_INTEGER;
+    #search_memo = new Map();
+    #break_memo = null;
+    /**
+     * Whether `#tokenize_text` is inside a list-item inline run: single
+     * newlines are soft breaks whose following structural indent strips from
+     * content, and paragraph-break/block-lookahead checks are skipped (the run
+     * scanner pre-validated every line in the run as continuation).
+     */
+    #list_run_strip = false;
+    constructor(text) {
+        this.#text = text;
+    }
+    /**
+     * Memoized `String#indexOf` over the immutable text.
+     *
+     * Delimiter and closing-tag searches scan forward from the current lex
+     * position, and a needle that never occurs again would otherwise rescan to
+     * the end of the text on every attempt — making inputs dense in unclosed
+     * candidates quadratic (e.g. `Vec<String>`-style generics searching for
+     * `</String>` at every occurrence). Caching the last result per needle
+     * makes each missing needle scan the remaining text at most once.
+     */
+    #index_of(needle, from) {
+        const memo = this.#search_memo.get(needle);
+        if (memo !== undefined && memo.from <= from && (memo.result === -1 || memo.result >= from)) {
+            return memo.result;
+        }
+        const result = this.#text.indexOf(needle, from);
+        if (memo === undefined) {
+            this.#search_memo.set(needle, { from, result });
+        }
+        else {
+            memo.from = from;
+            memo.result = result;
+        }
+        return result;
+    }
+    tokenize() {
+        // Skip leading blank lines
+        this.#skip_blank_lines();
+        while (this.#index < this.#text.length) {
+            // Check for block elements at column 0
+            if (this.#is_at_column_0()) {
+                if (this.#tokenize_heading())
+                    continue;
+                if (this.#tokenize_hr())
+                    continue;
+                if (this.#tokenize_codeblock())
+                    continue;
+                if (this.#tokenize_list())
+                    continue;
+                if (this.#tokenize_blockquote())
+                    continue;
+            }
+            // Check for paragraph break
+            if (this.#is_at_paragraph_break()) {
+                const start = this.#index;
+                this.#skip_blank_lines();
+                this.#tokens.push({ type: 'paragraph_break', start, end: this.#index });
+                continue;
+            }
+            // Inline tokenization
+            this.#tokenize_inline();
+        }
+        return this.#tokens;
+    }
+    #is_at_column_0() {
+        return this.#index === 0 || this.#text.charCodeAt(this.#index - 1) === NEWLINE;
+    }
+    /**
+     * Whether a paragraph break starts at the current position: a newline
+     * followed (across optional line whitespace) by another newline — i.e. the
+     * next line is blank. Linear in total despite the inner scan: the scan only
+     * runs at newline positions and covers each whitespace run at most once
+     * per newline preceding it.
+     */
+    #is_at_paragraph_break() {
+        if (this.#text.charCodeAt(this.#index) !== NEWLINE)
+            return false;
+        let i = this.#index + 1;
+        while (i < this.#text.length && is_line_whitespace(this.#text.charCodeAt(i))) {
+            i++;
+        }
+        return i < this.#text.length && this.#text.charCodeAt(i) === NEWLINE;
+    }
+    #match(str) {
+        return this.#text.startsWith(str, this.#index);
+    }
+    /**
+     * Skip newlines and whitespace-only lines (including a trailing
+     * whitespace-only line at EOF). Stops at the start of any line with
+     * content — leading whitespace on a content line stays in the text.
+     */
+    #skip_blank_lines() {
+        while (this.#index < this.#text.length) {
+            const char_code = this.#text.charCodeAt(this.#index);
+            if (char_code === NEWLINE) {
+                this.#index++;
+                continue;
+            }
+            if (!is_line_whitespace(char_code))
+                return;
+            let i = this.#index;
+            while (i < this.#text.length && is_line_whitespace(this.#text.charCodeAt(i))) {
+                i++;
+            }
+            if (i < this.#text.length && this.#text.charCodeAt(i) !== NEWLINE)
+                return; // line has content
+            this.#index = i; // blank to end of line (or EOF); newline consumed next iteration
+        }
+    }
+    // -- Block tokenizers --
+    #tokenize_heading() {
+        let i = this.#index;
+        // Count hashes (must be 1-6)
+        let hash_count = 0;
+        while (i < this.#text.length &&
+            this.#text.charCodeAt(i) === HASH &&
+            hash_count <= MAX_HEADING_LEVEL) {
+            hash_count++;
+            i++;
+        }
+        if (hash_count === 0 || hash_count > MAX_HEADING_LEVEL)
+            return false;
+        // Must have space after hashes
+        if (i >= this.#text.length || this.#text.charCodeAt(i) !== SPACE)
+            return false;
+        i++; // consume the space
+        // Must have at least one non-whitespace character after the space
+        let has_content = false;
+        for (let j = i; j < this.#text.length; j++) {
+            const char_code = this.#text.charCodeAt(j);
+            if (char_code === NEWLINE)
+                break;
+            if (!is_line_whitespace(char_code)) {
+                has_content = true;
+                break;
+            }
+        }
+        if (!has_content)
+            return false;
+        const start = this.#index;
+        // Advance past "## " (hashes + space)
+        this.#index = start + hash_count + 1;
+        this.#tokens.push({
+            type: 'heading_start',
+            level: hash_count,
+            start,
+            end: this.#index, // end of "## " prefix
+        });
+        // Find end-of-line to bound nested tokenizers (prevents tag scanner from scanning past heading)
+        let eol = this.#text.indexOf('\n', this.#index);
+        if (eol === -1)
+            eol = this.#text.length;
+        const saved_max = this.#max_search_index;
+        this.#max_search_index = eol;
+        // Tokenize inline content until newline or EOF
+        // tokenize_text may consume a newline as part of block-element lookahead,
+        // so we check emitted text tokens for embedded newlines and trim.
+        while (this.#index < eol) {
+            const token_count_before = this.#tokens.length;
+            this.#tokenize_inline();
+            // Check if the last emitted token is text containing a newline
+            if (this.#tokens.length > token_count_before) {
+                const last_token = this.#tokens[this.#tokens.length - 1];
+                if (last_token.type === 'text') {
+                    const newline_idx = last_token.content.indexOf('\n');
+                    if (newline_idx !== -1) {
+                        const trimmed = last_token.content.slice(0, newline_idx);
+                        if (trimmed) {
+                            last_token.content = trimmed;
+                            last_token.end = last_token.start + trimmed.length;
+                        }
+                        else {
+                            this.#tokens.pop();
+                        }
+                        this.#index = last_token.start + newline_idx;
+                        break;
+                    }
+                }
+            }
+        }
+        this.#max_search_index = saved_max;
+        // Emit heading_end marker so the token parser knows where heading content stops
+        this.#tokens.push({ type: 'heading_end', start: this.#index, end: this.#index });
+        // Skip newlines after heading
+        this.#skip_blank_lines();
+        return true;
+    }
+    #tokenize_hr() {
+        let i = this.#index;
+        // Must have exactly three hyphens
+        if (i + HR_HYPHEN_COUNT > this.#text.length ||
+            this.#text.charCodeAt(i) !== HYPHEN ||
+            this.#text.charCodeAt(i + 1) !== HYPHEN ||
+            this.#text.charCodeAt(i + 2) !== HYPHEN) {
+            return false;
+        }
+        i += HR_HYPHEN_COUNT;
+        // After the three hyphens, only whitespace and newline (or EOF) allowed
+        while (i < this.#text.length) {
+            const char_code = this.#text.charCodeAt(i);
+            if (char_code === NEWLINE)
+                break;
+            if (!is_line_whitespace(char_code))
+                return false;
+            i++;
+        }
+        const start = this.#index;
+        this.#index = i; // consume up to but not including newline
+        this.#tokens.push({ type: 'hr', start, end: this.#index });
+        // Skip newlines after HR
+        this.#skip_blank_lines();
+        return true;
+    }
+    #tokenize_codeblock() {
+        // a top-level fence is the indent-0 case of the general fence matcher
+        const fence = this.#match_fence(this.#index, 0);
+        if (fence === null)
+            return false;
+        this.#tokens.push(fence.token);
+        this.#index = fence.end;
+        // Skip newlines after codeblock
+        this.#skip_blank_lines();
+        return true;
+    }
+    // -- Blockquote tokenizers --
+    /**
+     * Match a quote prefix in the complete sync text — never `'pending'`.
+     * See `mdz_match_blockquote_prefix`.
+     */
+    #match_blockquote_prefix(i) {
+        const prefix = mdz_match_blockquote_prefix(this.#text, i, true);
+        return prefix === 'pending' ? null : prefix;
+    }
+    /**
+     * Whether the line at `i` opens a quote: a valid prefix with non-whitespace
+     * content after the full prefix on the same line (contentless prefix lines
+     * never start a quote — no empty start, mirroring lists).
+     */
+    #is_blockquote_opener(i) {
+        const prefix = this.#match_blockquote_prefix(i);
+        return (prefix !== null &&
+            mdz_blockquote_line_has_content(this.#text, prefix.content_start, true) === true);
+    }
+    /**
+     * Tokenize a blockquote whose opener line is at column 0. Returns false
+     * with no side effects when the line doesn't open one. Consumes trailing
+     * blank lines after the quote, mirroring the other block tokenizers.
+     */
+    #tokenize_blockquote() {
+        if (this.#text.charCodeAt(this.#index) !== RIGHT_ANGLE)
+            return false;
+        if (!this.#is_blockquote_opener(this.#index))
+            return false;
+        this.#tokenize_blockquote_at(this.#index, -1);
+        this.#skip_blank_lines();
+        return true;
+    }
+    /**
+     * Tokenize a blockquote starting at the validated opener position `first`.
+     * A quote's content is a mini-document behind the per-line prefix: the
+     * extent scan collects consecutive prefix lines (a blank or non-prefix
+     * line ends the quote — no lazy continuation), one prefix level strips
+     * from each line (deeper `>`s stay in the content for the nested document
+     * to parse), and the stripped content sub-lexes through a fresh `MdzLexer`
+     * with token positions remapped to source offsets. Nested quotes, lists,
+     * fences, and headings inside all follow from the recursion.
+     *
+     * `item_indent` is -1 for top-level quotes (continuation prefixes sit at
+     * column 0 exactly) or the attach item's marker indent for in-item quotes
+     * (continuation lines carry any indent deeper than it before their
+     * prefix). Leaves `#index` at the last quote line's end, before its
+     * newline (or at EOF).
+     */
+    #tokenize_blockquote_at(first, item_indent) {
+        // the inner document is the stripped source region including each line's
+        // terminating newline — only a source-EOF-cut final line lacks one, so a
+        // quote's content parses exactly like a standalone document (a complete
+        // fence opener line on the last quote line opens an empty fence, not text)
+        const parts = [];
+        const segments = [];
+        let inner_len = 0;
+        let line_first = first;
+        let last_line_end = first;
+        while (true) {
+            const strip = line_first + mdz_blockquote_strip_width(this.#text, line_first);
+            let nl = this.#text.indexOf('\n', line_first);
+            if (nl === -1)
+                nl = this.#text.length;
+            segments.push({ inner_start: inner_len, source_start: strip });
+            const content = this.#text.slice(Math.min(strip, nl), nl < this.#text.length ? nl + 1 : nl);
+            parts.push(content);
+            inner_len += content.length;
+            last_line_end = nl;
+            if (nl >= this.#text.length)
+                break;
+            // the next line continues the quote iff it carries a valid prefix at
+            // the required position: column 0 exactly for top-level quotes, any
+            // indent deeper than the item's marker for in-item quotes
+            const line_start = nl + 1;
+            let next_first = line_start;
+            if (item_indent >= 0) {
+                while (next_first < this.#text.length &&
+                    is_line_whitespace(this.#text.charCodeAt(next_first))) {
+                    next_first++;
+                }
+                if (next_first - line_start <= item_indent)
+                    break;
+            }
+            if (this.#text.charCodeAt(next_first) !== RIGHT_ANGLE)
+                break;
+            if (this.#match_blockquote_prefix(next_first) === null)
+                break;
+            line_first = next_first;
+        }
+        this.#tokens.push({ type: 'blockquote_open', start: first, end: first });
+        const inner_tokens = new MdzLexer(parts.join('')).tokenize();
+        for (const token of inner_tokens) {
+            token.start = mdz_remap_segments(segments, token.start);
+            token.end = mdz_remap_segments(segments, token.end);
+            this.#tokens.push(token);
+        }
+        this.#tokens.push({ type: 'blockquote_close', start: last_line_end, end: last_line_end });
+        this.#index = last_line_end;
+    }
+    // -- List tokenizers --
+    /**
+     * Tokenize an entire list starting at a column-0 marker line, emitting
+     * `list_open`/`list_item_open`/`list_item_close`/`list_close` structure
+     * tokens (plus `paragraph_break` and `codeblock` for in-item block
+     * children). Returns false with no side effects when the current line
+     * doesn't open a list: not a marker, an empty marker (empty items never
+     * start a list), or a non-`1.` ordered marker interrupting a paragraph.
+     */
+    #tokenize_list() {
+        const first_marker = this.#match_list_marker(this.#index);
+        if (first_marker === null || first_marker.empty)
+            return false;
+        // ordered interrupt guard: only `1.` opens an ordered list against a
+        // current paragraph, so hard-wrapped `2024. The…` lines stay prose
+        if (first_marker.ordered && first_marker.number !== 1 && this.#in_paragraph())
+            return false;
+        const levels = [];
+        this.#list_open(levels, 0, first_marker);
+        this.#list_run(levels, first_marker.content_start);
+        // line loop — each iteration starts with #index at the newline (or EOF)
+        // that terminated the previous run, block child, or empty-marker line
+        while (this.#index < this.#text.length) {
+            this.#index++; // consume the newline
+            // blank lines are contained, not terminal: scan to the next content
+            // line and let its shape decide (sibling/nest/paragraph-child/close)
+            let line_start = this.#index;
+            let first = line_start;
+            let at_eof = false;
+            while (true) {
+                let i = first;
+                while (i < this.#text.length && is_line_whitespace(this.#text.charCodeAt(i))) {
+                    i++;
+                }
+                if (i >= this.#text.length) {
+                    // blank to EOF — consume it; the list closes at EOF
+                    this.#index = i;
+                    at_eof = true;
+                    break;
+                }
+                if (this.#text.charCodeAt(i) === NEWLINE) {
+                    line_start = i + 1; // blank line — keep scanning
+                    first = line_start;
+                    continue;
+                }
+                first = i; // content line found
+                break;
+            }
+            if (at_eof)
+                break;
+            const indent = first - line_start;
+            const deepest = levels[levels.length - 1];
+            const marker = this.#match_list_marker(first);
+            if (marker !== null && !marker.empty) {
+                if (indent > deepest.indent) {
+                    // indenting past the deepest level nests (any ordered number here:
+                    // the `2024.`-guard lives in the run scanner, and reaching this
+                    // line means a blank or block boundary intervened)
+                    this.#list_open(levels, indent, marker);
+                }
+                else {
+                    // a dedent snaps to the nearest surviving level
+                    while (levels.length > 1 && levels[levels.length - 1].indent > indent) {
+                        this.#list_close(levels);
+                    }
+                    if (levels[levels.length - 1].ordered === marker.ordered) {
+                        // sibling item
+                        this.#tokens.push({ type: 'list_item_close', start: first, end: first });
+                        this.#list_item_open(marker);
+                    }
+                    else {
+                        // marker-type switch: close the list, open a new one at this indent
+                        this.#list_close(levels);
+                        this.#list_open(levels, indent, marker);
+                    }
+                }
+                this.#list_run(levels, marker.content_start);
+                continue;
+            }
+            if (marker !== null && marker.empty) {
+                // empty item: only at an indent exactly matching an open level of
+                // the same marker type — anything else degrades below
+                const level_idx = levels.findIndex((l) => l.indent === indent && l.ordered === marker.ordered);
+                if (level_idx !== -1) {
+                    while (levels.length - 1 > level_idx) {
+                        this.#list_close(levels);
+                    }
+                    this.#tokens.push({ type: 'list_item_close', start: first, end: first });
+                    this.#list_item_open(marker);
+                    // consume the marker and any trailing whitespace, leaving #index
+                    // at the line's newline (or EOF)
+                    let j = marker.marker_end;
+                    while (j < this.#text.length && is_line_whitespace(this.#text.charCodeAt(j))) {
+                        j++;
+                    }
+                    this.#index = j;
+                    continue;
+                }
+            }
+            // a deeper line that classifies as a valid fence open becomes a code
+            // block child of the item it would otherwise continue
+            if (indent > 0 && this.#text.charCodeAt(first) === BACKTICK) {
+                const target_idx = this.#list_continuation_target(levels, indent);
+                if (target_idx !== -1) {
+                    const fence = this.#match_fence(first, indent);
+                    if (fence !== null) {
+                        while (levels.length - 1 > target_idx) {
+                            this.#list_close(levels);
+                        }
+                        this.#tokens.push(fence.token);
+                        this.#index = fence.end;
+                        continue;
+                    }
+                }
+            }
+            // a deeper quote-opener line becomes a blockquote child of the item
+            // it would otherwise continue (the fence precedent — interrupts a
+            // current run without a blank line)
+            if (indent > 0 && this.#text.charCodeAt(first) === RIGHT_ANGLE) {
+                const target_idx = this.#list_continuation_target(levels, indent);
+                if (target_idx !== -1 && this.#is_blockquote_opener(first)) {
+                    while (levels.length - 1 > target_idx) {
+                        this.#list_close(levels);
+                    }
+                    this.#tokenize_blockquote_at(first, levels[levels.length - 1].indent);
+                    continue;
+                }
+            }
+            // any other line deeper than an open item's marker opens a new
+            // `Paragraph` block child of that item (the item's inline run is never
+            // current here — a blank, block child, or dedent intervened)
+            if (indent > 0) {
+                const target_idx = this.#list_continuation_target(levels, indent);
+                if (target_idx !== -1) {
+                    while (levels.length - 1 > target_idx) {
+                        this.#list_close(levels);
+                    }
+                    this.#tokens.push({ type: 'paragraph_break', start: line_start, end: line_start });
+                    this.#list_run(levels, first);
+                    continue;
+                }
+            }
+            // nothing list-shaped — the list ends; the main loop re-dispatches
+            // this line (paragraph, heading, HR, fence)
+            this.#index = line_start;
+            break;
+        }
+        while (levels.length > 0) {
+            this.#list_close(levels);
+        }
+        return true;
+    }
+    /**
+     * Match a list marker at `i`: `-` or 1-9 digits + `.`, followed by exactly
+     * one space and content — or by nothing but whitespace to end of line,
+     * which is an `empty` marker. Returns null for anything else (`-x`, `--`,
+     * a tab after the marker, a 10+ digit number).
+     */
+    #match_list_marker(i) {
+        const c = this.#text.charCodeAt(i);
+        if (c === HYPHEN) {
+            const after = i + 1;
+            if (this.#rest_is_blank(after)) {
+                return {
+                    ordered: false,
+                    number: 0,
+                    start: i,
+                    marker_end: after,
+                    content_start: after,
+                    empty: true,
+                };
+            }
+            if (this.#text.charCodeAt(after) !== SPACE)
+                return null;
+            const content_start = after + 1;
+            const empty = this.#rest_is_blank(content_start);
+            return { ordered: false, number: 0, start: i, marker_end: after, content_start, empty };
+        }
+        if (c >= ZERO && c <= NINE) {
+            let j = i + 1;
+            while (j < this.#text.length &&
+                this.#text.charCodeAt(j) >= ZERO &&
+                this.#text.charCodeAt(j) <= NINE) {
+                j++;
+            }
+            if (j - i > MAX_LIST_NUMBER_DIGITS)
+                return null;
+            if (j >= this.#text.length || this.#text.charCodeAt(j) !== PERIOD)
+                return null;
+            const number = Number(this.#text.slice(i, j));
+            const after = j + 1;
+            if (this.#rest_is_blank(after)) {
+                return {
+                    ordered: true,
+                    number,
+                    start: i,
+                    marker_end: after,
+                    content_start: after,
+                    empty: true,
+                };
+            }
+            if (this.#text.charCodeAt(after) !== SPACE)
+                return null;
+            const content_start = after + 1;
+            const empty = this.#rest_is_blank(content_start);
+            return { ordered: true, number, start: i, marker_end: after, content_start, empty };
+        }
+        return null;
+    }
+    /** Whether only line whitespace remains between `from` and the line's end (or EOF). */
+    #rest_is_blank(from) {
+        let i = from;
+        while (i < this.#text.length && is_line_whitespace(this.#text.charCodeAt(i))) {
+            i++;
+        }
+        return i >= this.#text.length || this.#text.charCodeAt(i) === NEWLINE;
+    }
+    /**
+     * Whether a paragraph's inline run is current — used by the ordered
+     * interrupt guard. True when the last emitted token isn't a block boundary.
+     */
+    #in_paragraph() {
+        if (this.#tokens.length === 0)
+            return false;
+        const type = this.#tokens[this.#tokens.length - 1].type;
+        return (type !== 'paragraph_break' &&
+            type !== 'hr' &&
+            type !== 'codeblock' &&
+            type !== 'heading_end' &&
+            type !== 'list_close' &&
+            type !== 'blockquote_close');
+    }
+    /** Open a new list level (list + first item) at `indent`. */
+    #list_open(levels, indent, marker) {
+        this.#tokens.push({
+            type: 'list_open',
+            ordered: marker.ordered,
+            start: marker.start,
+            end: marker.start,
+        });
+        this.#list_item_open(marker);
+        levels.push({ indent, ordered: marker.ordered });
+    }
+    /** Emit a `list_item_open` token for `marker`. */
+    #list_item_open(marker) {
+        const end = marker.empty ? marker.marker_end : marker.content_start;
+        if (marker.ordered) {
+            this.#tokens.push({ type: 'list_item_open', number: marker.number, start: marker.start, end });
+        }
+        else {
+            this.#tokens.push({ type: 'list_item_open', start: marker.start, end });
+        }
+    }
+    /** Close the deepest open level: its current item, then its list. */
+    #list_close(levels) {
+        this.#tokens.push({ type: 'list_item_close', start: this.#index, end: this.#index });
+        this.#tokens.push({ type: 'list_close', start: this.#index, end: this.#index });
+        levels.pop();
+    }
+    /**
+     * The level index of the deepest open item a line at `indent` can attach
+     * to — the deepest level with marker indent strictly less than `indent` —
+     * or -1. Levels above it get popped by the caller.
+     */
+    #list_continuation_target(levels, indent) {
+        for (let i = levels.length - 1; i >= 0; i--) {
+            if (levels[i].indent < indent)
+                return i;
+        }
+        return -1;
+    }
+    /**
+     * Tokenize one inline run (a marker line's content, or a paragraph block
+     * child) including its continuation lines. The run is a
+     * paragraph-equivalent region: `#max_search_index` bounds delimiter
+     * searches to the run's end, so inline pairs never cross item boundaries,
+     * and `#list_run_strip` makes single newlines soft breaks with the
+     * continuation indent stripped from content. Leaves `#index` at the run's
+     * terminating newline (or EOF).
+     */
+    #list_run(levels, content_start) {
+        // extra whitespace after the marker space is content-leading slop, trimmed
+        let i = content_start;
+        while (i < this.#text.length && is_line_whitespace(this.#text.charCodeAt(i))) {
+            i++;
+        }
+        this.#index = i;
+        const run_end = this.#list_run_end(i, levels[levels.length - 1].indent);
+        const saved_max = this.#max_search_index;
+        this.#max_search_index = run_end;
+        this.#list_run_strip = true;
+        while (this.#index < run_end) {
+            this.#tokenize_inline();
+        }
+        this.#list_run_strip = false;
+        this.#max_search_index = saved_max;
+    }
+    /**
+     * Scan forward from `from` for the end of the current inline run: the
+     * newline (or EOF) before the first line that isn't a continuation. A line
+     * continues the run when it's non-blank, indented deeper than the deepest
+     * open marker, and isn't a nesting marker or a fence opener line. A deeper
+     * non-`1.` ordered marker against the current run is continuation text
+     * (the hard-wrap `2024.` guard); a deeper empty marker is plain text.
+     */
+    #list_run_end(from, deepest_indent) {
+        let i = from;
+        while (true) {
+            const nl = this.#text.indexOf('\n', i);
+            if (nl === -1)
+                return this.#text.length;
+            const line_start = nl + 1;
+            let j = line_start;
+            while (j < this.#text.length && is_line_whitespace(this.#text.charCodeAt(j))) {
+                j++;
+            }
+            if (j >= this.#text.length || this.#text.charCodeAt(j) === NEWLINE)
+                return nl; // blank
+            const indent = j - line_start;
+            if (indent <= deepest_indent)
+                return nl; // sibling, dedent, or list end
+            const marker = this.#match_list_marker(j);
+            if (marker !== null && !marker.empty && (!marker.ordered || marker.number === 1)) {
+                return nl; // a nested list opens
+            }
+            if (this.#text.charCodeAt(j) === BACKTICK && this.#is_fence_opener(j)) {
+                return nl; // a fence opener line interrupts the run
+            }
+            if (this.#text.charCodeAt(j) === RIGHT_ANGLE && this.#is_blockquote_opener(j)) {
+                return nl; // a quote opener line interrupts the run
+            }
+            i = line_start; // continuation — keep scanning
+        }
+    }
+    /**
+     * Match a fenced code block whose opener is at `first` with the given
+     * `fence_indent` — 0 for top-level fences, the line's indent for in-item
+     * fences. A complete opener line always opens the fence, and raw mode is
+     * absolute: every following line is content until a closing fence (>= the
+     * opener's backticks, only trailing whitespace, at any indent <= the
+     * opener's) or EOF — a dedented or column-0 content line never
+     * force-closes, and an unclosed fence consumes to EOF (one trailing
+     * newline trimmed, matching closed fences' content shape). Content lines
+     * strip `min(fence_indent, line_indent)` leading whitespace. Returns null
+     * only when the opener line is malformed — including an opener cut off at
+     * EOF without its newline; `end` is the closer line's end before its
+     * newline, or EOF when unclosed.
+     */
+    #match_fence(first, fence_indent) {
+        let i = first;
+        let backtick_count = 0;
+        while (i < this.#text.length && this.#text.charCodeAt(i) === BACKTICK) {
+            backtick_count++;
+            i++;
+        }
+        if (backtick_count < MIN_CODEBLOCK_BACKTICKS)
+            return null;
+        const lang_start = i;
+        while (i < this.#text.length) {
+            const c = this.#text.charCodeAt(i);
+            if (c === NEWLINE || is_line_whitespace(c))
+                break;
+            i++;
+        }
+        const lang = i > lang_start ? this.#text.slice(lang_start, i) : null;
+        while (i < this.#text.length && is_line_whitespace(this.#text.charCodeAt(i))) {
+            i++;
+        }
+        if (i >= this.#text.length || this.#text.charCodeAt(i) !== NEWLINE)
+            return null;
+        i++; // past the opener's newline
+        const content_start = i;
+        const content_lines = [];
+        while (i <= this.#text.length) {
+            const line_start = i;
+            let j = i;
+            while (j < this.#text.length && is_line_whitespace(this.#text.charCodeAt(j))) {
+                j++;
+            }
+            const line_indent = j - line_start;
+            // closer probe: indent <= the opener's, a run of >= the opener's
+            // backticks, then only whitespace to end of line
+            if (line_indent <= fence_indent &&
+                j < this.#text.length &&
+                this.#text.charCodeAt(j) === BACKTICK) {
+                let k = j;
+                let count = 0;
+                while (k < this.#text.length && this.#text.charCodeAt(k) === BACKTICK) {
+                    count++;
+                    k++;
+                }
+                if (count >= backtick_count) {
+                    let m = k;
+                    while (m < this.#text.length && is_line_whitespace(this.#text.charCodeAt(m))) {
+                        m++;
+                    }
+                    if (m >= this.#text.length || this.#text.charCodeAt(m) === NEWLINE) {
+                        // at indent 0 nothing ever strips, so content is one slice
+                        // (dropping the newline before the closer line)
+                        const content = fence_indent === 0
+                            ? this.#text.slice(content_start, Math.max(content_start, line_start - 1))
+                            : content_lines.join('\n');
+                        return {
+                            token: { type: 'codeblock', lang, content, start: first, end: m },
+                            end: m,
+                        };
+                    }
+                }
+            }
+            // content line — strip the structural indent, never non-whitespace
+            const nl = this.#text.indexOf('\n', line_start);
+            if (fence_indent > 0) {
+                const line_end = nl === -1 ? this.#text.length : nl;
+                content_lines.push(this.#text.slice(line_start + Math.min(fence_indent, line_indent), line_end));
+            }
+            if (nl === -1)
+                break;
+            i = nl + 1;
+        }
+        // unclosed — raw mode is absolute, so the fence consumes to EOF; one
+        // trailing newline trims, matching closed fences' content shape
+        let content = fence_indent === 0 ? this.#text.slice(content_start) : content_lines.join('\n');
+        if (content.endsWith('\n'))
+            content = content.slice(0, -1);
+        return {
+            token: { type: 'codeblock', lang, content, start: first, end: this.#text.length },
+            end: this.#text.length,
+        };
+    }
+    /**
+     * Whether a complete fence opener line starts at `i`: a run of >=3
+     * backticks, an optional language word, then only whitespace up to a
+     * terminating newline — an opener cut off at EOF stays text. With
+     * raw-mode-absolute fences this alone decides fence-ness (`#match_fence`
+     * never fails on a complete opener), so run-end scans use this cheap
+     * probe instead of consuming the fence.
+     */
+    #is_fence_opener(i) {
+        let j = i;
+        let count = 0;
+        while (j < this.#text.length && this.#text.charCodeAt(j) === BACKTICK) {
+            count++;
+            j++;
+        }
+        if (count < MIN_CODEBLOCK_BACKTICKS)
+            return false;
+        // optional language word: non-whitespace run ends at whitespace or the newline
+        while (j < this.#text.length) {
+            const c = this.#text.charCodeAt(j);
+            if (c === NEWLINE || is_line_whitespace(c))
+                break;
+            j++;
+        }
+        while (j < this.#text.length && is_line_whitespace(this.#text.charCodeAt(j))) {
+            j++;
+        }
+        return j < this.#text.length && this.#text.charCodeAt(j) === NEWLINE;
+    }
+    // -- Inline tokenizers --
+    #tokenize_inline() {
+        const char_code = this.#text.charCodeAt(this.#index);
+        switch (char_code) {
+            case BACKTICK:
+                this.#tokenize_code();
+                return;
+            case ASTERISK:
+                this.#tokenize_double_delimiter('**', 'bold');
+                return;
+            case UNDERSCORE:
+                this.#tokenize_italic();
+                return;
+            case TILDE:
+                this.#tokenize_double_delimiter('~~', 'strikethrough');
+                return;
+            case LEFT_BRACKET:
+                this.#tokenize_markdown_link();
+                return;
+            case LEFT_ANGLE:
+                this.#tokenize_tag();
+                return;
+            default:
+                this.#tokenize_text();
+                return;
+        }
+    }
+    #tokenize_code() {
+        const start = this.#index;
+        this.#index++; // consume `
+        // Find closing backtick, stop at newline or search boundary
+        let content_end = -1;
+        const search_limit = Math.min(this.#max_search_index, this.#text.length);
+        for (let i = this.#index; i < search_limit; i++) {
+            const char_code = this.#text.charCodeAt(i);
+            if (char_code === BACKTICK) {
+                content_end = i;
+                break;
+            }
+            if (char_code === NEWLINE)
+                break;
+        }
+        if (content_end === -1) {
+            // Unclosed backtick
+            this.#emit_text('`', start);
+            return;
+        }
+        const content = this.#text.slice(this.#index, content_end);
+        // Empty inline code
+        if (content.length === 0) {
+            this.#index = content_end + 1;
+            this.#emit_text('``', start);
+            return;
+        }
+        this.#index = content_end + 1;
+        this.#tokens.push({ type: 'code', content, start, end: this.#index });
+    }
+    #tokenize_double_delimiter(delimiter, kind) {
+        const start = this.#index;
+        // Check for the doubled delimiter
+        if (!this.#match(delimiter)) {
+            // Single char - text
+            this.#index++;
+            this.#emit_text(delimiter[0], start);
+            return;
+        }
+        // Find closing delimiter within current search boundary
+        const search_end = Math.min(this.#max_search_index, this.#text.length);
+        let close_index = this.#index_of(delimiter, start + 2);
+        if (close_index !== -1 && close_index >= search_end) {
+            close_index = -1;
+        }
+        if (close_index === -1) {
+            // Unclosed
+            this.#index += 2;
+            this.#emit_text(delimiter, start);
+            return;
+        }
+        // Check for paragraph break between open and close
+        if (this.#has_paragraph_break_between(start + 2, close_index)) {
+            this.#index += 2;
+            this.#emit_text(delimiter, start);
+            return;
+        }
+        // Emit open token, tokenize children up to close, emit close token
+        this.#index += 2;
+        const open_type = kind === 'bold' ? 'bold_open' : 'strikethrough_open';
+        const close_type = kind === 'bold' ? 'bold_close' : 'strikethrough_close';
+        const open_token_index = this.#tokens.length;
+        this.#tokens.push({ type: open_type, start, end: this.#index });
+        // Set search boundary for nested parsers
+        const saved_max = this.#max_search_index;
+        this.#max_search_index = close_index;
+        // Tokenize children inline content up to close_index
+        while (this.#index < close_index) {
+            if (this.#is_at_paragraph_break())
+                break;
+            this.#tokenize_inline();
+        }
+        this.#max_search_index = saved_max;
+        if (this.#index === close_index && this.#match(delimiter)) {
+            // Check if empty
+            const has_children = open_token_index < this.#tokens.length - 1;
+            if (!has_children) {
+                // Empty - convert to text
+                this.#tokens.splice(open_token_index, 1);
+                this.#index = start;
+                this.#index += 4;
+                this.#emit_text(delimiter + delimiter, start);
+                return;
+            }
+            this.#tokens.push({ type: close_type, start: close_index, end: close_index + 2 });
+            this.#index = close_index + 2;
+        }
+        else {
+            // Didn't reach closing - convert opening to text
+            this.#tokens[open_token_index] = { type: 'text', content: delimiter, start, end: start + 2 };
+        }
+    }
+    #tokenize_italic() {
+        const start = this.#index;
+        // Check opening word boundary
+        if (!this.#is_at_word_boundary(this.#index, true, false)) {
+            this.#index++;
+            this.#emit_text('_', start);
+            return;
+        }
+        // Find closing delimiter within search boundary
+        const search_end = Math.min(this.#max_search_index, this.#text.length);
+        let close_index = this.#index_of('_', start + 1);
+        if (close_index !== -1 && close_index >= search_end) {
+            close_index = -1;
+        }
+        if (close_index === -1) {
+            this.#index++;
+            this.#emit_text('_', start);
+            return;
+        }
+        // Check closing word boundary
+        if (!this.#is_at_word_boundary(close_index + 1, false, true)) {
+            this.#index++;
+            this.#emit_text('_', start);
+            return;
+        }
+        // Check for paragraph break between
+        if (this.#has_paragraph_break_between(start + 1, close_index)) {
+            this.#index++;
+            this.#emit_text('_', start);
+            return;
+        }
+        // Emit open token
+        this.#index++;
+        const open_token_index = this.#tokens.length;
+        this.#tokens.push({ type: 'italic_open', start, end: this.#index });
+        // Set search boundary for nested parsers
+        const saved_max = this.#max_search_index;
+        this.#max_search_index = close_index;
+        // Tokenize children up to close_index
+        while (this.#index < close_index) {
+            if (this.#is_at_paragraph_break())
+                break;
+            this.#tokenize_inline();
+        }
+        this.#max_search_index = saved_max;
+        if (this.#index === close_index && this.#match('_')) {
+            // Check if empty
+            const has_children = open_token_index < this.#tokens.length - 1;
+            if (!has_children) {
+                // Empty - convert to text
+                this.#tokens.splice(open_token_index, 1);
+                this.#index = start;
+                this.#index += 2;
+                this.#emit_text('__', start);
+                return;
+            }
+            this.#tokens.push({
+                type: 'italic_close',
+                start: close_index,
+                end: close_index + 1,
+            });
+            this.#index = close_index + 1;
+        }
+        else {
+            // Convert opening to text
+            this.#tokens[open_token_index] = {
+                type: 'text',
+                content: '_',
+                start,
+                end: start + 1,
+            };
+        }
+    }
+    #tokenize_markdown_link() {
+        const start = this.#index;
+        this.#index++; // consume [ (dispatch guarantees it)
+        // Emit link_text_open
+        this.#tokens.push({ type: 'link_text_open', start, end: this.#index });
+        // Tokenize children until ] — only `]` delimits link text; a bare `)`
+        // is ordinary content (matching the streaming parser, which has no `)`
+        // dispatch case — the reference scan finds its own `)` after `](`)
+        while (this.#index < this.#text.length) {
+            if (this.#text.charCodeAt(this.#index) === RIGHT_BRACKET)
+                break;
+            if (this.#is_at_paragraph_break())
+                break;
+            this.#tokenize_inline();
+        }
+        // Check for ]
+        if (this.#index >= this.#text.length || this.#text.charCodeAt(this.#index) !== RIGHT_BRACKET) {
+            // Revert - remove link_text_open and all children tokens added
+            this.#revert_tokens_from_link_open(start);
+            return;
+        }
+        const bracket_close_start = this.#index;
+        this.#index++; // consume ]
+        this.#tokens.push({
+            type: 'link_text_close',
+            start: bracket_close_start,
+            end: this.#index,
+        });
+        // Check for (
+        if (this.#index >= this.#text.length || this.#text.charCodeAt(this.#index) !== LEFT_PAREN) {
+            this.#revert_tokens_from_link_open(start);
+            return;
+        }
+        this.#index++; // consume (
+        // Find closing )
+        const close_paren = this.#index_of(')', this.#index);
+        if (close_paren === -1) {
+            this.#revert_tokens_from_link_open(start);
+            return;
+        }
+        const reference = this.#text.slice(this.#index, close_paren);
+        // Validate reference
+        if (!reference.trim()) {
+            this.#revert_tokens_from_link_open(start);
+            return;
+        }
+        // Validate all characters
+        for (let i = 0; i < reference.length; i++) {
+            const char_code = reference.charCodeAt(i);
+            if (!is_valid_path_char(char_code)) {
+                this.#revert_tokens_from_link_open(start);
+                return;
+            }
+        }
+        // Reject unsafe protocols (javascript:, data:, etc.) — `is_valid_path_char`
+        // permits `:` so we need an explicit filter here.
+        if (!mdz_is_safe_reference(reference)) {
+            this.#revert_tokens_from_link_open(start);
+            return;
+        }
+        this.#index = close_paren + 1;
+        const link_type = mdz_is_url(reference) ? 'external' : 'internal';
+        this.#tokens.push({
+            type: 'link_ref',
+            reference,
+            link_type,
+            start: bracket_close_start + 1, // after ]
+            end: this.#index,
+        });
+    }
+    #revert_tokens_from_link_open(start) {
+        // Find and remove link_text_open and all tokens after it
+        let open_idx = -1;
+        for (let i = this.#tokens.length - 1; i >= 0; i--) {
+            if (this.#tokens[i].type === 'link_text_open' && this.#tokens[i].start === start) {
+                open_idx = i;
+                break;
+            }
+        }
+        if (open_idx !== -1) {
+            this.#tokens.splice(open_idx);
+        }
+        this.#index = start + 1;
+        this.#emit_text('[', start);
+    }
+    #tokenize_tag() {
+        const start = this.#index;
+        this.#index++; // consume <
+        // Tag name must start with a letter
+        if (this.#index >= this.#text.length || !is_letter(this.#text.charCodeAt(this.#index))) {
+            this.#emit_text('<', start);
+            return;
+        }
+        // Collect tag name
+        const tag_name_start = this.#index;
+        while (this.#index < this.#text.length &&
+            is_tag_name_char(this.#text.charCodeAt(this.#index))) {
+            this.#index++;
+        }
+        // non-empty: the is_letter check above guarantees at least one name char
+        const tag_name = this.#text.slice(tag_name_start, this.#index);
+        const first_char_code = tag_name.charCodeAt(0);
+        const is_component = first_char_code >= A_UPPER && first_char_code <= Z_UPPER;
+        // Skip whitespace
+        while (this.#index < this.#text.length && this.#text.charCodeAt(this.#index) === SPACE) {
+            this.#index++;
+        }
+        // Check for self-closing />
+        if (this.#index + 1 < this.#text.length &&
+            this.#text.charCodeAt(this.#index) === SLASH &&
+            this.#text.charCodeAt(this.#index + 1) === RIGHT_ANGLE) {
+            this.#index += 2;
+            this.#tokens.push({
+                type: 'tag_self_close',
+                name: tag_name,
+                is_component,
+                start,
+                end: this.#index,
+            });
+            return;
+        }
+        // Check for >
+        if (this.#index >= this.#text.length || this.#text.charCodeAt(this.#index) !== RIGHT_ANGLE) {
+            this.#index = start + 1;
+            this.#emit_text('<', start);
+            return;
+        }
+        this.#index++; // consume >
+        // Check for closing tag existence before committing —
+        // must exist within search boundary and before any paragraph break.
+        // A closer at exactly `#max_search_index` is allowed: that bound can
+        // only coincide with a closing-tag occurrence when an enclosing
+        // same-name tag set it (the nesting handoff — this tag consumes the
+        // closer and the enclosing tag re-verifies its own next iteration);
+        // every other bound (heading EOL, list run end, delimiter close
+        // position) points at a character that can't begin a closing tag.
+        const closing_tag = `</${tag_name}>`;
+        const search_limit = Math.min(this.#max_search_index, this.#text.length);
+        const closing_tag_pos = this.#index_of(closing_tag, this.#index);
+        if (closing_tag_pos === -1 || closing_tag_pos > search_limit) {
+            this.#index = start + 1;
+            this.#emit_text('<', start);
+            return;
+        }
+        if (this.#has_paragraph_break_between(this.#index, closing_tag_pos)) {
+            this.#index = start + 1;
+            this.#emit_text('<', start);
+            return;
+        }
+        // Emit tag_open
+        const open_token_index = this.#tokens.length;
+        this.#tokens.push({ type: 'tag_open', name: tag_name, is_component, start, end: this.#index });
+        // Tokenize children until the closing tag. Each iteration narrows
+        // `#max_search_index` to the next closer so a child's delimiter scan
+        // can never cross it (`` <b>`a</b>x` `` must not swallow `</b>` into
+        // the code span). Recomputed per iteration because a nested same-name
+        // tag legitimately consumes the nearest closer (`<b>x<b>y</b></b>`).
+        const saved_max = this.#max_search_index;
+        while (this.#index < this.#text.length) {
+            if (this.#match(closing_tag)) {
+                this.#max_search_index = saved_max;
+                const close_start = this.#index;
+                this.#index += closing_tag.length;
+                this.#tokens.push({
+                    type: 'tag_close',
+                    name: tag_name,
+                    start: close_start,
+                    end: this.#index,
+                });
+                return;
+            }
+            const next_close = this.#index_of(closing_tag, this.#index);
+            if (next_close === -1 ||
+                next_close > Math.min(saved_max, this.#text.length) ||
+                this.#has_paragraph_break_between(this.#index, next_close)) {
+                break; // a nested same-name tag consumed the verified closer
+            }
+            this.#max_search_index = next_close;
+            this.#tokenize_inline();
+        }
+        // No valid closer left — revert: drop the open token and all consumed
+        // children tokens, emit `<` as text, and re-lex from the next char
+        // (the `#revert_tokens_from_link_open` pattern; the old fallthrough
+        // left a dangling `tag_open` whose parser fallback discarded content)
+        this.#max_search_index = saved_max;
+        this.#tokens.splice(open_token_index);
+        this.#index = start + 1;
+        this.#emit_text('<', start);
+    }
+    #tokenize_text() {
+        const start = this.#index;
+        // Check for URL or internal path at current position
+        if (this.#is_at_url()) {
+            this.#tokenize_auto_link_url();
+            return;
+        }
+        if (is_at_absolute_path(this.#text, this.#index)) {
+            this.#tokenize_auto_link_internal();
+            return;
+        }
+        if (is_at_relative_path(this.#text, this.#index)) {
+            this.#tokenize_auto_link_internal();
+            return;
+        }
+        // in a list-item run, continuation-line structural indent strips from
+        // content: parts accumulate around each stripped region
+        let parts = null;
+        let part_start = start;
+        while (this.#index < this.#text.length) {
+            const char_code = this.#text.charCodeAt(this.#index);
+            // Stop at special characters
+            if (char_code === BACKTICK ||
+                char_code === ASTERISK ||
+                char_code === UNDERSCORE ||
+                char_code === TILDE ||
+                char_code === LEFT_BRACKET ||
+                char_code === RIGHT_BRACKET ||
+                char_code === LEFT_ANGLE) {
+                break;
+            }
+            if (char_code === NEWLINE) {
+                if (this.#list_run_strip) {
+                    if (this.#index >= this.#max_search_index)
+                        break; // the run's terminator
+                    // soft break: keep the newline, strip the next line's leading
+                    // whitespace (structural indent, never content)
+                    this.#index++;
+                    const ws_start = this.#index;
+                    let i = ws_start;
+                    while (i < this.#text.length && is_line_whitespace(this.#text.charCodeAt(i))) {
+                        i++;
+                    }
+                    if (i > ws_start) {
+                        (parts ??= []).push(this.#text.slice(part_start, ws_start));
+                        part_start = i;
+                        this.#index = i;
+                    }
+                    continue;
+                }
+                // Check for paragraph break
+                if (this.#is_at_paragraph_break())
+                    break;
+                // When next line could start a block element, consume the newline and
+                // stop (`1` covers ordered list markers — only `1.` interrupts)
+                const next_i = this.#index + 1;
+                if (next_i < this.#text.length) {
+                    const next_char = this.#text.charCodeAt(next_i);
+                    if (next_char === HASH ||
+                        next_char === HYPHEN ||
+                        next_char === BACKTICK ||
+                        next_char === ONE ||
+                        next_char === RIGHT_ANGLE) {
+                        this.#index++; // consume the newline
+                        break;
+                    }
+                }
+            }
+            // Check for URL or internal path mid-text (char code guard avoids startsWith on every char)
+            if (((char_code === 104 /* h */ || char_code === 72) /* H */ && this.#is_at_url()) ||
+                (char_code === SLASH && is_at_absolute_path(this.#text, this.#index)) ||
+                (char_code === PERIOD && is_at_relative_path(this.#text, this.#index))) {
+                break;
+            }
+            this.#index++;
+        }
+        // Ensure we always consume at least one character
+        if (this.#index === start && this.#index < this.#text.length) {
+            this.#index++;
+        }
+        const content = parts === null
+            ? this.#text.slice(start, this.#index)
+            : parts.join('') + this.#text.slice(part_start, this.#index);
+        this.#tokens.push({ type: 'text', content, start, end: this.#index });
+    }
+    // -- Auto-link tokenizers --
+    #tokenize_auto_link_url() {
+        const start = this.#index;
+        // Consume protocol (case-insensitive match; original casing preserved in reference)
+        this.#index += match_url_prefix_case_insensitive(this.#text, this.#index);
+        // Collect URL characters
+        while (this.#index < this.#text.length) {
+            const char_code = this.#text.charCodeAt(this.#index);
+            if (char_code === SPACE || char_code === NEWLINE || !is_valid_path_char(char_code)) {
+                break;
+            }
+            this.#index++;
+        }
+        let reference = this.#text.slice(start, this.#index);
+        reference = mdz_trim_url_punctuation(reference);
+        this.#index = start + reference.length;
+        this.#tokens.push({
+            type: 'autolink',
+            reference,
+            link_type: 'external',
+            start,
+            end: this.#index,
+        });
+    }
+    #tokenize_auto_link_internal() {
+        const start = this.#index;
+        // Collect path characters
+        while (this.#index < this.#text.length) {
+            const char_code = this.#text.charCodeAt(this.#index);
+            if (char_code === SPACE || char_code === NEWLINE || !is_valid_path_char(char_code)) {
+                break;
+            }
+            this.#index++;
+        }
+        let reference = this.#text.slice(start, this.#index);
+        reference = mdz_trim_url_punctuation(reference);
+        this.#index = start + reference.length;
+        this.#tokens.push({
+            type: 'autolink',
+            reference,
+            link_type: 'internal',
+            start,
+            end: this.#index,
+        });
+    }
+    // -- Helper methods --
+    #emit_text(content, start) {
+        this.#tokens.push({ type: 'text', content, start, end: start + content.length });
+    }
+    #has_paragraph_break_between(from, to) {
+        // memoized search — a per-candidate char loop would rescan the same
+        // region for every delimiter/tag candidate in a long paragraph. The
+        // break can't straddle `to`: `to` is a found delimiter/tag position
+        // (never whitespace or a newline), so a break starting before it
+        // completes before it.
+        const idx = this.#next_paragraph_break(from);
+        return idx !== -1 && idx < to;
+    }
+    /**
+     * Position of the next paragraph break at or after `from` — a newline
+     * whose following line is blank (whitespace-only) — or `-1`. Memoized like
+     * `#index_of`: the text is immutable, so the first break at or after a
+     * scanned origin stays valid for any later origin it still follows.
+     */
+    #next_paragraph_break(from) {
+        const memo = this.#break_memo;
+        if (memo !== null && memo.from <= from && (memo.result === -1 || memo.result >= from)) {
+            return memo.result;
+        }
+        let result = -1;
+        let i = this.#text.indexOf('\n', from);
+        while (i !== -1) {
+            let j = i + 1;
+            while (j < this.#text.length && is_line_whitespace(this.#text.charCodeAt(j))) {
+                j++;
+            }
+            if (j < this.#text.length && this.#text.charCodeAt(j) === NEWLINE) {
+                result = i;
+                break;
+            }
+            // no break here; the next candidate newline is past the scanned run
+            i = this.#text.indexOf('\n', j);
+        }
+        if (memo === null) {
+            this.#break_memo = { from, result };
+        }
+        else {
+            memo.from = from;
+            memo.result = result;
+        }
+        return result;
+    }
+    #is_at_url() {
+        // word boundary check: skip when preceded by [A-Za-z0-9] — `xhttps://...`
+        // is plain text, matching the streaming parser (false negatives over
+        // false positives)
+        if (this.#index > 0 && is_word_char(this.#text.charCodeAt(this.#index - 1))) {
+            return false;
+        }
+        // Scheme matching is case-insensitive (RFC 3986).
+        const prefix_len = match_url_prefix_case_insensitive(this.#text, this.#index);
+        if (prefix_len === 0)
+            return false;
+        if (this.#index + prefix_len >= this.#text.length)
+            return false;
+        const next_char = this.#text.charCodeAt(this.#index + prefix_len);
+        return next_char !== SPACE && next_char !== NEWLINE;
+    }
+    /**
+     * Word-boundary check for the italic delimiter (the only delimiter with
+     * boundary rules — `**`/`~~` pair anywhere). `_` is self-opaque — it
+     * counts as a word char in its own boundary checks, so runs like
+     * `__init__` never form open/close pairs — while other delimiters stay
+     * transparent (so `_~~x~~_` nests).
+     */
+    #is_at_word_boundary(index, check_before, check_after) {
+        if (check_before && index > 0) {
+            const prev = this.#text.charCodeAt(index - 1);
+            if (is_word_char(prev) || prev === UNDERSCORE)
+                return false;
+        }
+        if (check_after && index < this.#text.length) {
+            const next = this.#text.charCodeAt(index);
+            if (is_word_char(next) || next === UNDERSCORE)
+                return false;
+        }
+        return true;
+    }
+}