@fuzdev/fuz_ui 0.204.0 → 0.205.0

package/dist/mdz_lexer.js

@@ -1,732 +0,0 @@
-/**
- * mdz lexer — tokenizes input into a flat `MdzToken[]` stream.
- *
- * Phase 1 of the two-phase lexer+parser alternative to the single-pass parser
- * in `mdz.ts`. Phase 2 is in `mdz_token_parser.ts`.
- *
- * @module
- */
-import { mdz_is_url, mdz_is_safe_reference, is_letter, is_tag_name_char, is_word_char, is_valid_path_char, trim_trailing_punctuation, BACKTICK, ASTERISK, UNDERSCORE, TILDE, NEWLINE, HYPHEN, HASH, SPACE, TAB, LEFT_ANGLE, RIGHT_ANGLE, SLASH, PERIOD, LEFT_BRACKET, LEFT_PAREN, RIGHT_PAREN, RIGHT_BRACKET, A_UPPER, Z_UPPER, HR_HYPHEN_COUNT, MIN_CODEBLOCK_BACKTICKS, MAX_HEADING_LEVEL, match_url_prefix_case_insensitive, is_at_absolute_path, is_at_relative_path, } from './mdz_helpers.js';
-//
-// Lexer
-//
-export class MdzLexer {
-    #text;
-    #index = 0;
-    #tokens = [];
-    #max_search_index = Number.MAX_SAFE_INTEGER;
-    constructor(text) {
-        this.#text = text;
-    }
-    tokenize() {
-        // Skip leading newlines
-        this.#skip_newlines();
-        while (this.#index < this.#text.length) {
-            // Check for block elements at column 0
-            if (this.#is_at_column_0()) {
-                if (this.#tokenize_heading())
-                    continue;
-                if (this.#tokenize_hr())
-                    continue;
-                if (this.#tokenize_codeblock())
-                    continue;
-            }
-            // Check for paragraph break
-            if (this.#is_at_paragraph_break()) {
-                const start = this.#index;
-                this.#skip_newlines();
-                this.#tokens.push({ type: 'paragraph_break', start, end: this.#index });
-                continue;
-            }
-            // Inline tokenization
-            this.#tokenize_inline();
-        }
-        return this.#tokens;
-    }
-    #is_at_column_0() {
-        return this.#index === 0 || this.#text.charCodeAt(this.#index - 1) === NEWLINE;
-    }
-    #is_at_paragraph_break() {
-        return (this.#index + 1 < this.#text.length &&
-            this.#text.charCodeAt(this.#index) === NEWLINE &&
-            this.#text.charCodeAt(this.#index + 1) === NEWLINE);
-    }
-    #match(str) {
-        return this.#text.startsWith(str, this.#index);
-    }
-    #skip_newlines() {
-        while (this.#index < this.#text.length && this.#text.charCodeAt(this.#index) === NEWLINE) {
-            this.#index++;
-        }
-    }
-    // -- Block tokenizers --
-    #tokenize_heading() {
-        let i = this.#index;
-        // Count hashes (must be 1-6)
-        let hash_count = 0;
-        while (i < this.#text.length &&
-            this.#text.charCodeAt(i) === HASH &&
-            hash_count <= MAX_HEADING_LEVEL) {
-            hash_count++;
-            i++;
-        }
-        if (hash_count === 0 || hash_count > MAX_HEADING_LEVEL)
-            return false;
-        // Must have space after hashes
-        if (i >= this.#text.length || this.#text.charCodeAt(i) !== SPACE)
-            return false;
-        i++; // consume the space
-        // Must have at least one non-whitespace character after the space
-        let has_content = false;
-        for (let j = i; j < this.#text.length; j++) {
-            const char_code = this.#text.charCodeAt(j);
-            if (char_code === NEWLINE)
-                break;
-            if (char_code !== SPACE && char_code !== TAB) {
-                has_content = true;
-                break;
-            }
-        }
-        if (!has_content)
-            return false;
-        const start = this.#index;
-        // Advance past "## " (hashes + space)
-        this.#index = start + hash_count + 1;
-        this.#tokens.push({
-            type: 'heading_start',
-            level: hash_count,
-            start,
-            end: this.#index, // end of "## " prefix
-        });
-        // Find end-of-line to bound nested tokenizers (prevents tag scanner from scanning past heading)
-        let eol = this.#text.indexOf('\n', this.#index);
-        if (eol === -1)
-            eol = this.#text.length;
-        const saved_max = this.#max_search_index;
-        this.#max_search_index = eol;
-        // Tokenize inline content until newline or EOF
-        // tokenize_text may consume a newline as part of block-element lookahead,
-        // so we check emitted text tokens for embedded newlines and trim.
-        while (this.#index < eol) {
-            const token_count_before = this.#tokens.length;
-            this.#tokenize_inline();
-            // Check if the last emitted token is text containing a newline
-            if (this.#tokens.length > token_count_before) {
-                const last_token = this.#tokens[this.#tokens.length - 1];
-                if (last_token.type === 'text') {
-                    const newline_idx = last_token.content.indexOf('\n');
-                    if (newline_idx !== -1) {
-                        const trimmed = last_token.content.slice(0, newline_idx);
-                        if (trimmed) {
-                            last_token.content = trimmed;
-                            last_token.end = last_token.start + trimmed.length;
-                        }
-                        else {
-                            this.#tokens.pop();
-                        }
-                        this.#index = last_token.start + newline_idx;
-                        break;
-                    }
-                }
-            }
-        }
-        this.#max_search_index = saved_max;
-        // Emit heading_end marker so the token parser knows where heading content stops
-        this.#tokens.push({ type: 'heading_end', start: this.#index, end: this.#index });
-        // Skip newlines after heading (like original parser's main loop)
-        this.#skip_newlines();
-        return true;
-    }
-    #tokenize_hr() {
-        let i = this.#index;
-        // Must have exactly three hyphens
-        if (i + HR_HYPHEN_COUNT > this.#text.length ||
-            this.#text.charCodeAt(i) !== HYPHEN ||
-            this.#text.charCodeAt(i + 1) !== HYPHEN ||
-            this.#text.charCodeAt(i + 2) !== HYPHEN) {
-            return false;
-        }
-        i += HR_HYPHEN_COUNT;
-        // After the three hyphens, only whitespace and newline (or EOF) allowed
-        while (i < this.#text.length) {
-            const char_code = this.#text.charCodeAt(i);
-            if (char_code === NEWLINE)
-                break;
-            if (char_code !== SPACE)
-                return false;
-            i++;
-        }
-        const start = this.#index;
-        this.#index = i; // consume up to but not including newline
-        this.#tokens.push({ type: 'hr', start, end: this.#index });
-        // Skip newlines after HR
-        this.#skip_newlines();
-        return true;
-    }
-    #tokenize_codeblock() {
-        let i = this.#index;
-        // Count backticks
-        let backtick_count = 0;
-        while (i < this.#text.length && this.#text.charCodeAt(i) === BACKTICK) {
-            backtick_count++;
-            i++;
-        }
-        if (backtick_count < MIN_CODEBLOCK_BACKTICKS)
-            return false;
-        // Skip optional language hint
-        const lang_start = i;
-        while (i < this.#text.length) {
-            const char_code = this.#text.charCodeAt(i);
-            if (char_code === SPACE || char_code === NEWLINE)
-                break;
-            i++;
-        }
-        const lang = i > lang_start ? this.#text.slice(lang_start, i) : null;
-        // Skip trailing spaces on opening fence line
-        while (i < this.#text.length && this.#text.charCodeAt(i) === SPACE) {
-            i++;
-        }
-        // Must have newline after opening fence
-        if (i >= this.#text.length || this.#text.charCodeAt(i) !== NEWLINE)
-            return false;
-        i++; // consume the newline
-        const content_start = i;
-        // Search for closing fence
-        const closing_fence = '`'.repeat(backtick_count);
-        while (i < this.#text.length) {
-            if (this.#text.startsWith(closing_fence, i)) {
-                const prev_char = i > 0 ? this.#text.charCodeAt(i - 1) : NEWLINE;
-                if (prev_char === NEWLINE || i === 0) {
-                    const content = this.#text.slice(content_start, i);
-                    const final_content = content.endsWith('\n') ? content.slice(0, -1) : content;
-                    if (final_content.length === 0)
-                        return false; // Empty code block
-                    // Verify closing fence line
-                    let j = i + backtick_count;
-                    while (j < this.#text.length && this.#text.charCodeAt(j) === SPACE) {
-                        j++;
-                    }
-                    if (j < this.#text.length && this.#text.charCodeAt(j) !== NEWLINE) {
-                        return false;
-                    }
-                    const start = this.#index;
-                    this.#index = j; // advance past closing fence and trailing spaces
-                    this.#tokens.push({
-                        type: 'codeblock',
-                        lang,
-                        content: final_content,
-                        start,
-                        end: this.#index,
-                    });
-                    // Skip newlines after codeblock
-                    this.#skip_newlines();
-                    return true;
-                }
-            }
-            i++;
-        }
-        return false; // No closing fence found
-    }
-    // -- Inline tokenizers --
-    #tokenize_inline() {
-        const char_code = this.#text.charCodeAt(this.#index);
-        switch (char_code) {
-            case BACKTICK:
-                this.#tokenize_code();
-                return;
-            case ASTERISK:
-                this.#tokenize_bold();
-                return;
-            case UNDERSCORE:
-                this.#tokenize_single_delimiter('_', 'italic');
-                return;
-            case TILDE:
-                this.#tokenize_single_delimiter('~', 'strikethrough');
-                return;
-            case LEFT_BRACKET:
-                this.#tokenize_markdown_link();
-                return;
-            case LEFT_ANGLE:
-                this.#tokenize_tag();
-                return;
-            default:
-                this.#tokenize_text();
-                return;
-        }
-    }
-    #tokenize_code() {
-        const start = this.#index;
-        this.#index++; // consume `
-        // Find closing backtick, stop at newline or search boundary
-        let content_end = -1;
-        const search_limit = Math.min(this.#max_search_index, this.#text.length);
-        for (let i = this.#index; i < search_limit; i++) {
-            const char_code = this.#text.charCodeAt(i);
-            if (char_code === BACKTICK) {
-                content_end = i;
-                break;
-            }
-            if (char_code === NEWLINE)
-                break;
-        }
-        if (content_end === -1) {
-            // Unclosed backtick
-            this.#emit_text('`', start);
-            return;
-        }
-        const content = this.#text.slice(this.#index, content_end);
-        // Empty inline code
-        if (content.length === 0) {
-            this.#index = content_end + 1;
-            this.#emit_text('``', start);
-            return;
-        }
-        this.#index = content_end + 1;
-        this.#tokens.push({ type: 'code', content, start, end: this.#index });
-    }
-    #tokenize_bold() {
-        const start = this.#index;
-        // Check for **
-        if (!this.#match('**')) {
-            // Single asterisk - text
-            this.#index++;
-            this.#emit_text('*', start);
-            return;
-        }
-        // Find closing ** within current search boundary
-        const search_end = Math.min(this.#max_search_index, this.#text.length);
-        let close_index = this.#text.indexOf('**', start + 2);
-        if (close_index !== -1 && close_index >= search_end) {
-            close_index = -1;
-        }
-        if (close_index === -1) {
-            // Unclosed **
-            this.#index += 2;
-            this.#emit_text('**', start);
-            return;
-        }
-        // Check for paragraph break between open and close
-        if (this.#has_paragraph_break_between(start + 2, close_index)) {
-            this.#index += 2;
-            this.#emit_text('**', start);
-            return;
-        }
-        // Emit bold_open, tokenize children up to close, emit bold_close
-        this.#index += 2;
-        const open_token_index = this.#tokens.length;
-        this.#tokens.push({ type: 'bold_open', start, end: this.#index });
-        // Set search boundary for nested parsers
-        const saved_max = this.#max_search_index;
-        this.#max_search_index = close_index;
-        // Tokenize children inline content up to close_index
-        while (this.#index < close_index) {
-            if (this.#is_at_paragraph_break())
-                break;
-            this.#tokenize_inline();
-        }
-        this.#max_search_index = saved_max;
-        if (this.#index === close_index && this.#match('**')) {
-            // Check if empty
-            const has_children = open_token_index < this.#tokens.length - 1;
-            if (!has_children) {
-                // Empty bold - convert to text
-                this.#tokens.splice(open_token_index, 1);
-                this.#index = start;
-                this.#index += 4;
-                this.#emit_text('****', start);
-                return;
-            }
-            this.#tokens.push({ type: 'bold_close', start: close_index, end: close_index + 2 });
-            this.#index = close_index + 2;
-        }
-        else {
-            // Didn't reach closing - convert opening to text
-            this.#tokens[open_token_index] = { type: 'text', content: '**', start, end: start + 2 };
-        }
-    }
-    #tokenize_single_delimiter(delimiter, kind) {
-        const start = this.#index;
-        // Check opening word boundary
-        if (!this.#is_at_word_boundary(this.#index, true, false)) {
-            this.#index++;
-            this.#emit_text(delimiter, start);
-            return;
-        }
-        // Find closing delimiter within search boundary
-        const search_end = Math.min(this.#max_search_index, this.#text.length);
-        let close_index = this.#text.indexOf(delimiter, start + 1);
-        if (close_index !== -1 && close_index >= search_end) {
-            close_index = -1;
-        }
-        if (close_index === -1) {
-            this.#index++;
-            this.#emit_text(delimiter, start);
-            return;
-        }
-        // Check closing word boundary
-        if (!this.#is_at_word_boundary(close_index + 1, false, true)) {
-            this.#index++;
-            this.#emit_text(delimiter, start);
-            return;
-        }
-        // Check for paragraph break between
-        if (this.#has_paragraph_break_between(start + 1, close_index)) {
-            this.#index++;
-            this.#emit_text(delimiter, start);
-            return;
-        }
-        // Emit open token
-        this.#index++;
-        const open_type = kind === 'italic' ? 'italic_open' : 'strikethrough_open';
-        const close_type = kind === 'italic' ? 'italic_close' : 'strikethrough_close';
-        const open_token_index = this.#tokens.length;
-        this.#tokens.push({ type: open_type, start, end: this.#index });
-        // Set search boundary for nested parsers
-        const saved_max = this.#max_search_index;
-        this.#max_search_index = close_index;
-        // Tokenize children up to close_index
-        while (this.#index < close_index) {
-            if (this.#is_at_paragraph_break())
-                break;
-            this.#tokenize_inline();
-        }
-        this.#max_search_index = saved_max;
-        if (this.#index === close_index && this.#match(delimiter)) {
-            // Check if empty
-            const has_children = open_token_index < this.#tokens.length - 1;
-            if (!has_children) {
-                // Empty - convert to text
-                this.#tokens.splice(open_token_index, 1);
-                this.#index = start;
-                this.#index += 2;
-                this.#emit_text(delimiter + delimiter, start);
-                return;
-            }
-            this.#tokens.push({
-                type: close_type,
-                start: close_index,
-                end: close_index + 1,
-            });
-            this.#index = close_index + 1;
-        }
-        else {
-            // Convert opening to text
-            this.#tokens[open_token_index] = {
-                type: 'text',
-                content: delimiter,
-                start,
-                end: start + 1,
-            };
-        }
-    }
-    #tokenize_markdown_link() {
-        const start = this.#index;
-        // Consume [
-        if (this.#text.charCodeAt(this.#index) !== LEFT_BRACKET) {
-            this.#index++;
-            this.#emit_text(this.#text[start], start);
-            return;
-        }
-        this.#index++;
-        // Emit link_text_open
-        this.#tokens.push({ type: 'link_text_open', start, end: this.#index });
-        // Tokenize children until ]
-        while (this.#index < this.#text.length) {
-            if (this.#text.charCodeAt(this.#index) === RIGHT_BRACKET)
-                break;
-            if (this.#is_at_paragraph_break())
-                break;
-            // Stop at ] and ) as delimiters
-            if (this.#text.charCodeAt(this.#index) === RIGHT_PAREN)
-                break;
-            this.#tokenize_inline();
-        }
-        // Check for ]
-        if (this.#index >= this.#text.length || this.#text.charCodeAt(this.#index) !== RIGHT_BRACKET) {
-            // Revert - remove link_text_open and all children tokens added
-            this.#revert_tokens_from_link_open(start);
-            return;
-        }
-        const bracket_close_start = this.#index;
-        this.#index++; // consume ]
-        this.#tokens.push({
-            type: 'link_text_close',
-            start: bracket_close_start,
-            end: this.#index,
-        });
-        // Check for (
-        if (this.#index >= this.#text.length || this.#text.charCodeAt(this.#index) !== LEFT_PAREN) {
-            this.#revert_tokens_from_link_open(start);
-            return;
-        }
-        this.#index++; // consume (
-        // Find closing )
-        const close_paren = this.#text.indexOf(')', this.#index);
-        if (close_paren === -1) {
-            this.#revert_tokens_from_link_open(start);
-            return;
-        }
-        const reference = this.#text.slice(this.#index, close_paren);
-        // Validate reference
-        if (!reference.trim()) {
-            this.#revert_tokens_from_link_open(start);
-            return;
-        }
-        // Validate all characters
-        for (let i = 0; i < reference.length; i++) {
-            const char_code = reference.charCodeAt(i);
-            if (!is_valid_path_char(char_code)) {
-                this.#revert_tokens_from_link_open(start);
-                return;
-            }
-        }
-        // Reject unsafe protocols (javascript:, data:, etc.) — `is_valid_path_char`
-        // permits `:` so we need an explicit filter here.
-        if (!mdz_is_safe_reference(reference)) {
-            this.#revert_tokens_from_link_open(start);
-            return;
-        }
-        this.#index = close_paren + 1;
-        const link_type = mdz_is_url(reference) ? 'external' : 'internal';
-        this.#tokens.push({
-            type: 'link_ref',
-            reference,
-            link_type,
-            start: bracket_close_start + 1, // after ]
-            end: this.#index,
-        });
-    }
-    #revert_tokens_from_link_open(start) {
-        // Find and remove link_text_open and all tokens after it
-        let open_idx = -1;
-        for (let i = this.#tokens.length - 1; i >= 0; i--) {
-            if (this.#tokens[i].type === 'link_text_open' && this.#tokens[i].start === start) {
-                open_idx = i;
-                break;
-            }
-        }
-        if (open_idx !== -1) {
-            this.#tokens.splice(open_idx);
-        }
-        this.#index = start + 1;
-        this.#emit_text('[', start);
-    }
-    #tokenize_tag() {
-        const start = this.#index;
-        this.#index++; // consume <
-        // Tag name must start with a letter
-        if (this.#index >= this.#text.length || !is_letter(this.#text.charCodeAt(this.#index))) {
-            this.#emit_text('<', start);
-            return;
-        }
-        // Collect tag name
-        const tag_name_start = this.#index;
-        while (this.#index < this.#text.length &&
-            is_tag_name_char(this.#text.charCodeAt(this.#index))) {
-            this.#index++;
-        }
-        const tag_name = this.#text.slice(tag_name_start, this.#index);
-        if (tag_name.length === 0) {
-            this.#emit_text('<', start);
-            return;
-        }
-        const first_char_code = tag_name.charCodeAt(0);
-        const is_component = first_char_code >= A_UPPER && first_char_code <= Z_UPPER;
-        // Skip whitespace
-        while (this.#index < this.#text.length && this.#text.charCodeAt(this.#index) === SPACE) {
-            this.#index++;
-        }
-        // Check for self-closing />
-        if (this.#index + 1 < this.#text.length &&
-            this.#text.charCodeAt(this.#index) === SLASH &&
-            this.#text.charCodeAt(this.#index + 1) === RIGHT_ANGLE) {
-            this.#index += 2;
-            this.#tokens.push({
-                type: 'tag_self_close',
-                name: tag_name,
-                is_component,
-                start,
-                end: this.#index,
-            });
-            return;
-        }
-        // Check for >
-        if (this.#index >= this.#text.length || this.#text.charCodeAt(this.#index) !== RIGHT_ANGLE) {
-            this.#index = start + 1;
-            this.#emit_text('<', start);
-            return;
-        }
-        this.#index++; // consume >
-        // Check for closing tag existence before committing —
-        // must exist within search boundary and before any paragraph break
-        const closing_tag = `</${tag_name}>`;
-        const search_limit = Math.min(this.#max_search_index, this.#text.length);
-        const closing_tag_pos = this.#text.indexOf(closing_tag, this.#index);
-        if (closing_tag_pos === -1 || closing_tag_pos >= search_limit) {
-            this.#index = start + 1;
-            this.#emit_text('<', start);
-            return;
-        }
-        if (this.#has_paragraph_break_between(this.#index, closing_tag_pos)) {
-            this.#index = start + 1;
-            this.#emit_text('<', start);
-            return;
-        }
-        // Emit tag_open
-        this.#tokens.push({ type: 'tag_open', name: tag_name, is_component, start, end: this.#index });
-        // Tokenize children until closing tag
-        while (this.#index < this.#text.length) {
-            if (this.#match(closing_tag)) {
-                const close_start = this.#index;
-                this.#index += closing_tag.length;
-                this.#tokens.push({
-                    type: 'tag_close',
-                    name: tag_name,
-                    start: close_start,
-                    end: this.#index,
-                });
-                return;
-            }
-            this.#tokenize_inline();
-        }
-        // Shouldn't reach here since we verified closing tag exists
-        // But if we do, the tag_open token stays and parser will handle it
-    }
-    #tokenize_text() {
-        const start = this.#index;
-        // Check for URL or internal path at current position
-        if (this.#is_at_url()) {
-            this.#tokenize_auto_link_url();
-            return;
-        }
-        if (is_at_absolute_path(this.#text, this.#index)) {
-            this.#tokenize_auto_link_internal();
-            return;
-        }
-        if (is_at_relative_path(this.#text, this.#index)) {
-            this.#tokenize_auto_link_internal();
-            return;
-        }
-        while (this.#index < this.#text.length) {
-            const char_code = this.#text.charCodeAt(this.#index);
-            // Stop at special characters
-            if (char_code === BACKTICK ||
-                char_code === ASTERISK ||
-                char_code === UNDERSCORE ||
-                char_code === TILDE ||
-                char_code === LEFT_BRACKET ||
-                char_code === RIGHT_BRACKET ||
-                char_code === RIGHT_PAREN ||
-                char_code === LEFT_ANGLE) {
-                break;
-            }
-            // Check for paragraph break
-            if (this.#is_at_paragraph_break())
-                break;
-            // When next line could start a block element, consume the newline and stop
-            if (char_code === NEWLINE) {
-                const next_i = this.#index + 1;
-                if (next_i < this.#text.length) {
-                    const next_char = this.#text.charCodeAt(next_i);
-                    if (next_char === HASH || next_char === HYPHEN || next_char === BACKTICK) {
-                        this.#index++; // consume the newline
-                        break;
-                    }
-                }
-            }
-            // Check for URL or internal path mid-text (char code guard avoids startsWith on every char)
-            if (((char_code === 104 /* h */ || char_code === 72) /* H */ && this.#is_at_url()) ||
-                (char_code === SLASH && is_at_absolute_path(this.#text, this.#index)) ||
-                (char_code === PERIOD && is_at_relative_path(this.#text, this.#index))) {
-                break;
-            }
-            this.#index++;
-        }
-        // Ensure we always consume at least one character
-        if (this.#index === start && this.#index < this.#text.length) {
-            this.#index++;
-        }
-        const content = this.#text.slice(start, this.#index);
-        this.#emit_text(content, start);
-    }
-    // -- Auto-link tokenizers --
-    #tokenize_auto_link_url() {
-        const start = this.#index;
-        // Consume protocol (case-insensitive match; original casing preserved in reference)
-        this.#index += match_url_prefix_case_insensitive(this.#text, this.#index);
-        // Collect URL characters
-        while (this.#index < this.#text.length) {
-            const char_code = this.#text.charCodeAt(this.#index);
-            if (char_code === SPACE || char_code === NEWLINE || !is_valid_path_char(char_code)) {
-                break;
-            }
-            this.#index++;
-        }
-        let reference = this.#text.slice(start, this.#index);
-        reference = trim_trailing_punctuation(reference);
-        this.#index = start + reference.length;
-        this.#tokens.push({
-            type: 'autolink',
-            reference,
-            link_type: 'external',
-            start,
-            end: this.#index,
-        });
-    }
-    #tokenize_auto_link_internal() {
-        const start = this.#index;
-        // Collect path characters
-        while (this.#index < this.#text.length) {
-            const char_code = this.#text.charCodeAt(this.#index);
-            if (char_code === SPACE || char_code === NEWLINE || !is_valid_path_char(char_code)) {
-                break;
-            }
-            this.#index++;
-        }
-        let reference = this.#text.slice(start, this.#index);
-        reference = trim_trailing_punctuation(reference);
-        this.#index = start + reference.length;
-        this.#tokens.push({
-            type: 'autolink',
-            reference,
-            link_type: 'internal',
-            start,
-            end: this.#index,
-        });
-    }
-    // -- Helper methods --
-    #emit_text(content, start) {
-        this.#tokens.push({ type: 'text', content, start, end: start + content.length });
-    }
-    #has_paragraph_break_between(from, to) {
-        for (let i = from; i < to - 1; i++) {
-            if (this.#text.charCodeAt(i) === NEWLINE && this.#text.charCodeAt(i + 1) === NEWLINE) {
-                return true;
-            }
-        }
-        return false;
-    }
-    #is_at_url() {
-        // Scheme matching is case-insensitive (RFC 3986).
-        const prefix_len = match_url_prefix_case_insensitive(this.#text, this.#index);
-        if (prefix_len === 0)
-            return false;
-        if (this.#index + prefix_len >= this.#text.length)
-            return false;
-        const next_char = this.#text.charCodeAt(this.#index + prefix_len);
-        return next_char !== SPACE && next_char !== NEWLINE;
-    }
-    #is_at_word_boundary(index, check_before, check_after) {
-        if (check_before && index > 0) {
-            const prev = this.#text.charCodeAt(index - 1);
-            if (is_word_char(prev))
-                return false;
-        }
-        if (check_after && index < this.#text.length) {
-            const next = this.#text.charCodeAt(index);
-            if (is_word_char(next))
-                return false;
-        }
-        return true;
-    }
-}