@fuzdev/fuz_ui 0.204.0 → 0.205.0

package/src/lib/mdz_lexer.ts

@@ -1,1012 +0,0 @@
-/**
- * mdz lexer — tokenizes input into a flat `MdzToken[]` stream.
- *
- * Phase 1 of the two-phase lexer+parser alternative to the single-pass parser
- * in `mdz.ts`. Phase 2 is in `mdz_token_parser.ts`.
- *
- * @module
- */
-
-import {
-	mdz_is_url,
-	mdz_is_safe_reference,
-	is_letter,
-	is_tag_name_char,
-	is_word_char,
-	is_valid_path_char,
-	trim_trailing_punctuation,
-	BACKTICK,
-	ASTERISK,
-	UNDERSCORE,
-	TILDE,
-	NEWLINE,
-	HYPHEN,
-	HASH,
-	SPACE,
-	TAB,
-	LEFT_ANGLE,
-	RIGHT_ANGLE,
-	SLASH,
-	PERIOD,
-	LEFT_BRACKET,
-	LEFT_PAREN,
-	RIGHT_PAREN,
-	RIGHT_BRACKET,
-	A_UPPER,
-	Z_UPPER,
-	HR_HYPHEN_COUNT,
-	MIN_CODEBLOCK_BACKTICKS,
-	MAX_HEADING_LEVEL,
-	match_url_prefix_case_insensitive,
-	is_at_absolute_path,
-	is_at_relative_path,
-} from './mdz_helpers.js';
-
-//
-// Token types
-//
-
-export interface MdzTokenBase {
-	start: number;
-	end: number;
-}
-
-export type MdzToken =
-	| MdzTokenText
-	| MdzTokenCode
-	| MdzTokenCodeblock
-	| MdzTokenBoldOpen
-	| MdzTokenBoldClose
-	| MdzTokenItalicOpen
-	| MdzTokenItalicClose
-	| MdzTokenStrikethroughOpen
-	| MdzTokenStrikethroughClose
-	| MdzTokenLinkTextOpen
-	| MdzTokenLinkTextClose
-	| MdzTokenLinkRef
-	| MdzTokenAutolink
-	| MdzTokenHeadingStart
-	| MdzTokenHr
-	| MdzTokenTagOpen
-	| MdzTokenTagSelfClose
-	| MdzTokenTagClose
-	| MdzTokenHeadingEnd
-	| MdzTokenParagraphBreak;
-
-export interface MdzTokenText extends MdzTokenBase {
-	type: 'text';
-	content: string;
-}
-
-export interface MdzTokenCode extends MdzTokenBase {
-	type: 'code';
-	content: string;
-}
-
-export interface MdzTokenCodeblock extends MdzTokenBase {
-	type: 'codeblock';
-	lang: string | null;
-	content: string;
-}
-
-export interface MdzTokenBoldOpen extends MdzTokenBase {
-	type: 'bold_open';
-}
-
-export interface MdzTokenBoldClose extends MdzTokenBase {
-	type: 'bold_close';
-}
-
-export interface MdzTokenItalicOpen extends MdzTokenBase {
-	type: 'italic_open';
-}
-
-export interface MdzTokenItalicClose extends MdzTokenBase {
-	type: 'italic_close';
-}
-
-export interface MdzTokenStrikethroughOpen extends MdzTokenBase {
-	type: 'strikethrough_open';
-}
-
-export interface MdzTokenStrikethroughClose extends MdzTokenBase {
-	type: 'strikethrough_close';
-}
-
-export interface MdzTokenLinkTextOpen extends MdzTokenBase {
-	type: 'link_text_open';
-}
-
-export interface MdzTokenLinkTextClose extends MdzTokenBase {
-	type: 'link_text_close';
-}
-
-export interface MdzTokenLinkRef extends MdzTokenBase {
-	type: 'link_ref';
-	reference: string;
-	link_type: 'external' | 'internal';
-}
-
-export interface MdzTokenAutolink extends MdzTokenBase {
-	type: 'autolink';
-	reference: string;
-	link_type: 'external' | 'internal';
-}
-
-export interface MdzTokenHeadingStart extends MdzTokenBase {
-	type: 'heading_start';
-	level: 1 | 2 | 3 | 4 | 5 | 6;
-}
-
-export interface MdzTokenHr extends MdzTokenBase {
-	type: 'hr';
-}
-
-export interface MdzTokenTagOpen extends MdzTokenBase {
-	type: 'tag_open';
-	name: string;
-	is_component: boolean;
-}
-
-export interface MdzTokenTagSelfClose extends MdzTokenBase {
-	type: 'tag_self_close';
-	name: string;
-	is_component: boolean;
-}
-
-export interface MdzTokenTagClose extends MdzTokenBase {
-	type: 'tag_close';
-	name: string;
-}
-
-export interface MdzTokenHeadingEnd extends MdzTokenBase {
-	type: 'heading_end';
-}
-
-export interface MdzTokenParagraphBreak extends MdzTokenBase {
-	type: 'paragraph_break';
-}
-
-//
-// Lexer
-//
-
-export class MdzLexer {
-	#text: string;
-	#index: number = 0;
-	#tokens: Array<MdzToken> = [];
-	#max_search_index: number = Number.MAX_SAFE_INTEGER;
-
-	constructor(text: string) {
-		this.#text = text;
-	}
-
-	tokenize(): Array<MdzToken> {
-		// Skip leading newlines
-		this.#skip_newlines();
-
-		while (this.#index < this.#text.length) {
-			// Check for block elements at column 0
-			if (this.#is_at_column_0()) {
-				if (this.#tokenize_heading()) continue;
-				if (this.#tokenize_hr()) continue;
-				if (this.#tokenize_codeblock()) continue;
-			}
-
-			// Check for paragraph break
-			if (this.#is_at_paragraph_break()) {
-				const start = this.#index;
-				this.#skip_newlines();
-				this.#tokens.push({type: 'paragraph_break', start, end: this.#index});
-				continue;
-			}
-
-			// Inline tokenization
-			this.#tokenize_inline();
-		}
-
-		return this.#tokens;
-	}
-
-	#is_at_column_0(): boolean {
-		return this.#index === 0 || this.#text.charCodeAt(this.#index - 1) === NEWLINE;
-	}
-
-	#is_at_paragraph_break(): boolean {
-		return (
-			this.#index + 1 < this.#text.length &&
-			this.#text.charCodeAt(this.#index) === NEWLINE &&
-			this.#text.charCodeAt(this.#index + 1) === NEWLINE
-		);
-	}
-
-	#match(str: string): boolean {
-		return this.#text.startsWith(str, this.#index);
-	}
-
-	#skip_newlines(): void {
-		while (this.#index < this.#text.length && this.#text.charCodeAt(this.#index) === NEWLINE) {
-			this.#index++;
-		}
-	}
-
-	// -- Block tokenizers --
-
-	#tokenize_heading(): boolean {
-		let i = this.#index;
-
-		// Count hashes (must be 1-6)
-		let hash_count = 0;
-		while (
-			i < this.#text.length &&
-			this.#text.charCodeAt(i) === HASH &&
-			hash_count <= MAX_HEADING_LEVEL
-		) {
-			hash_count++;
-			i++;
-		}
-
-		if (hash_count === 0 || hash_count > MAX_HEADING_LEVEL) return false;
-
-		// Must have space after hashes
-		if (i >= this.#text.length || this.#text.charCodeAt(i) !== SPACE) return false;
-		i++; // consume the space
-
-		// Must have at least one non-whitespace character after the space
-		let has_content = false;
-		for (let j = i; j < this.#text.length; j++) {
-			const char_code = this.#text.charCodeAt(j);
-			if (char_code === NEWLINE) break;
-			if (char_code !== SPACE && char_code !== TAB) {
-				has_content = true;
-				break;
-			}
-		}
-
-		if (!has_content) return false;
-
-		const start = this.#index;
-		// Advance past "## " (hashes + space)
-		this.#index = start + hash_count + 1;
-
-		this.#tokens.push({
-			type: 'heading_start',
-			level: hash_count as 1 | 2 | 3 | 4 | 5 | 6,
-			start,
-			end: this.#index, // end of "## " prefix
-		});
-
-		// Find end-of-line to bound nested tokenizers (prevents tag scanner from scanning past heading)
-		let eol = this.#text.indexOf('\n', this.#index);
-		if (eol === -1) eol = this.#text.length;
-
-		const saved_max = this.#max_search_index;
-		this.#max_search_index = eol;
-
-		// Tokenize inline content until newline or EOF
-		// tokenize_text may consume a newline as part of block-element lookahead,
-		// so we check emitted text tokens for embedded newlines and trim.
-		while (this.#index < eol) {
-			const token_count_before = this.#tokens.length;
-			this.#tokenize_inline();
-
-			// Check if the last emitted token is text containing a newline
-			if (this.#tokens.length > token_count_before) {
-				const last_token = this.#tokens[this.#tokens.length - 1]!;
-				if (last_token.type === 'text') {
-					const newline_idx = last_token.content.indexOf('\n');
-					if (newline_idx !== -1) {
-						const trimmed = last_token.content.slice(0, newline_idx);
-						if (trimmed) {
-							last_token.content = trimmed;
-							last_token.end = last_token.start + trimmed.length;
-						} else {
-							this.#tokens.pop();
-						}
-						this.#index = last_token.start + newline_idx;
-						break;
-					}
-				}
-			}
-		}
-
-		this.#max_search_index = saved_max;
-
-		// Emit heading_end marker so the token parser knows where heading content stops
-		this.#tokens.push({type: 'heading_end', start: this.#index, end: this.#index});
-
-		// Skip newlines after heading (like original parser's main loop)
-		this.#skip_newlines();
-
-		return true;
-	}
-
-	#tokenize_hr(): boolean {
-		let i = this.#index;
-
-		// Must have exactly three hyphens
-		if (
-			i + HR_HYPHEN_COUNT > this.#text.length ||
-			this.#text.charCodeAt(i) !== HYPHEN ||
-			this.#text.charCodeAt(i + 1) !== HYPHEN ||
-			this.#text.charCodeAt(i + 2) !== HYPHEN
-		) {
-			return false;
-		}
-		i += HR_HYPHEN_COUNT;
-
-		// After the three hyphens, only whitespace and newline (or EOF) allowed
-		while (i < this.#text.length) {
-			const char_code = this.#text.charCodeAt(i);
-			if (char_code === NEWLINE) break;
-			if (char_code !== SPACE) return false;
-			i++;
-		}
-
-		const start = this.#index;
-		this.#index = i; // consume up to but not including newline
-
-		this.#tokens.push({type: 'hr', start, end: this.#index});
-
-		// Skip newlines after HR
-		this.#skip_newlines();
-
-		return true;
-	}
-
-	#tokenize_codeblock(): boolean {
-		let i = this.#index;
-
-		// Count backticks
-		let backtick_count = 0;
-		while (i < this.#text.length && this.#text.charCodeAt(i) === BACKTICK) {
-			backtick_count++;
-			i++;
-		}
-
-		if (backtick_count < MIN_CODEBLOCK_BACKTICKS) return false;
-
-		// Skip optional language hint
-		const lang_start = i;
-		while (i < this.#text.length) {
-			const char_code = this.#text.charCodeAt(i);
-			if (char_code === SPACE || char_code === NEWLINE) break;
-			i++;
-		}
-		const lang = i > lang_start ? this.#text.slice(lang_start, i) : null;
-
-		// Skip trailing spaces on opening fence line
-		while (i < this.#text.length && this.#text.charCodeAt(i) === SPACE) {
-			i++;
-		}
-
-		// Must have newline after opening fence
-		if (i >= this.#text.length || this.#text.charCodeAt(i) !== NEWLINE) return false;
-		i++; // consume the newline
-
-		const content_start = i;
-
-		// Search for closing fence
-		const closing_fence = '`'.repeat(backtick_count);
-		while (i < this.#text.length) {
-			if (this.#text.startsWith(closing_fence, i)) {
-				const prev_char = i > 0 ? this.#text.charCodeAt(i - 1) : NEWLINE;
-				if (prev_char === NEWLINE || i === 0) {
-					const content = this.#text.slice(content_start, i);
-					const final_content = content.endsWith('\n') ? content.slice(0, -1) : content;
-					if (final_content.length === 0) return false; // Empty code block
-
-					// Verify closing fence line
-					let j = i + backtick_count;
-					while (j < this.#text.length && this.#text.charCodeAt(j) === SPACE) {
-						j++;
-					}
-					if (j < this.#text.length && this.#text.charCodeAt(j) !== NEWLINE) {
-						return false;
-					}
-
-					const start = this.#index;
-					this.#index = j; // advance past closing fence and trailing spaces
-
-					this.#tokens.push({
-						type: 'codeblock',
-						lang,
-						content: final_content,
-						start,
-						end: this.#index,
-					});
-
-					// Skip newlines after codeblock
-					this.#skip_newlines();
-
-					return true;
-				}
-			}
-			i++;
-		}
-
-		return false; // No closing fence found
-	}
-
-	// -- Inline tokenizers --
-
-	#tokenize_inline(): void {
-		const char_code = this.#text.charCodeAt(this.#index);
-
-		switch (char_code) {
-			case BACKTICK:
-				this.#tokenize_code();
-				return;
-			case ASTERISK:
-				this.#tokenize_bold();
-				return;
-			case UNDERSCORE:
-				this.#tokenize_single_delimiter('_', 'italic');
-				return;
-			case TILDE:
-				this.#tokenize_single_delimiter('~', 'strikethrough');
-				return;
-			case LEFT_BRACKET:
-				this.#tokenize_markdown_link();
-				return;
-			case LEFT_ANGLE:
-				this.#tokenize_tag();
-				return;
-			default:
-				this.#tokenize_text();
-				return;
-		}
-	}
-
-	#tokenize_code(): void {
-		const start = this.#index;
-		this.#index++; // consume `
-
-		// Find closing backtick, stop at newline or search boundary
-		let content_end = -1;
-		const search_limit = Math.min(this.#max_search_index, this.#text.length);
-		for (let i = this.#index; i < search_limit; i++) {
-			const char_code = this.#text.charCodeAt(i);
-			if (char_code === BACKTICK) {
-				content_end = i;
-				break;
-			}
-			if (char_code === NEWLINE) break;
-		}
-
-		if (content_end === -1) {
-			// Unclosed backtick
-			this.#emit_text('`', start);
-			return;
-		}
-
-		const content = this.#text.slice(this.#index, content_end);
-
-		// Empty inline code
-		if (content.length === 0) {
-			this.#index = content_end + 1;
-			this.#emit_text('``', start);
-			return;
-		}
-
-		this.#index = content_end + 1;
-		this.#tokens.push({type: 'code', content, start, end: this.#index});
-	}
-
-	#tokenize_bold(): void {
-		const start = this.#index;
-
-		// Check for **
-		if (!this.#match('**')) {
-			// Single asterisk - text
-			this.#index++;
-			this.#emit_text('*', start);
-			return;
-		}
-
-		// Find closing ** within current search boundary
-		const search_end = Math.min(this.#max_search_index, this.#text.length);
-		let close_index = this.#text.indexOf('**', start + 2);
-		if (close_index !== -1 && close_index >= search_end) {
-			close_index = -1;
-		}
-		if (close_index === -1) {
-			// Unclosed **
-			this.#index += 2;
-			this.#emit_text('**', start);
-			return;
-		}
-
-		// Check for paragraph break between open and close
-		if (this.#has_paragraph_break_between(start + 2, close_index)) {
-			this.#index += 2;
-			this.#emit_text('**', start);
-			return;
-		}
-
-		// Emit bold_open, tokenize children up to close, emit bold_close
-		this.#index += 2;
-		const open_token_index = this.#tokens.length;
-		this.#tokens.push({type: 'bold_open', start, end: this.#index});
-
-		// Set search boundary for nested parsers
-		const saved_max = this.#max_search_index;
-		this.#max_search_index = close_index;
-
-		// Tokenize children inline content up to close_index
-		while (this.#index < close_index) {
-			if (this.#is_at_paragraph_break()) break;
-			this.#tokenize_inline();
-		}
-
-		this.#max_search_index = saved_max;
-
-		if (this.#index === close_index && this.#match('**')) {
-			// Check if empty
-			const has_children = open_token_index < this.#tokens.length - 1;
-
-			if (!has_children) {
-				// Empty bold - convert to text
-				this.#tokens.splice(open_token_index, 1);
-				this.#index = start;
-				this.#index += 4;
-				this.#emit_text('****', start);
-				return;
-			}
-
-			this.#tokens.push({type: 'bold_close', start: close_index, end: close_index + 2});
-			this.#index = close_index + 2;
-		} else {
-			// Didn't reach closing - convert opening to text
-			this.#tokens[open_token_index] = {type: 'text', content: '**', start, end: start + 2};
-		}
-	}
-
-	#tokenize_single_delimiter(delimiter: '_' | '~', kind: 'italic' | 'strikethrough'): void {
-		const start = this.#index;
-
-		// Check opening word boundary
-		if (!this.#is_at_word_boundary(this.#index, true, false)) {
-			this.#index++;
-			this.#emit_text(delimiter, start);
-			return;
-		}
-
-		// Find closing delimiter within search boundary
-		const search_end = Math.min(this.#max_search_index, this.#text.length);
-		let close_index = this.#text.indexOf(delimiter, start + 1);
-		if (close_index !== -1 && close_index >= search_end) {
-			close_index = -1;
-		}
-		if (close_index === -1) {
-			this.#index++;
-			this.#emit_text(delimiter, start);
-			return;
-		}
-
-		// Check closing word boundary
-		if (!this.#is_at_word_boundary(close_index + 1, false, true)) {
-			this.#index++;
-			this.#emit_text(delimiter, start);
-			return;
-		}
-
-		// Check for paragraph break between
-		if (this.#has_paragraph_break_between(start + 1, close_index)) {
-			this.#index++;
-			this.#emit_text(delimiter, start);
-			return;
-		}
-
-		// Emit open token
-		this.#index++;
-		const open_type = kind === 'italic' ? 'italic_open' : 'strikethrough_open';
-		const close_type = kind === 'italic' ? 'italic_close' : 'strikethrough_close';
-		const open_token_index = this.#tokens.length;
-		this.#tokens.push({type: open_type, start, end: this.#index});
-
-		// Set search boundary for nested parsers
-		const saved_max = this.#max_search_index;
-		this.#max_search_index = close_index;
-
-		// Tokenize children up to close_index
-		while (this.#index < close_index) {
-			if (this.#is_at_paragraph_break()) break;
-			this.#tokenize_inline();
-		}
-
-		this.#max_search_index = saved_max;
-
-		if (this.#index === close_index && this.#match(delimiter)) {
-			// Check if empty
-			const has_children = open_token_index < this.#tokens.length - 1;
-
-			if (!has_children) {
-				// Empty - convert to text
-				this.#tokens.splice(open_token_index, 1);
-				this.#index = start;
-				this.#index += 2;
-				this.#emit_text(delimiter + delimiter, start);
-				return;
-			}
-
-			this.#tokens.push({
-				type: close_type,
-				start: close_index,
-				end: close_index + 1,
-			});
-			this.#index = close_index + 1;
-		} else {
-			// Convert opening to text
-			this.#tokens[open_token_index] = {
-				type: 'text',
-				content: delimiter,
-				start,
-				end: start + 1,
-			};
-		}
-	}
-
-	#tokenize_markdown_link(): void {
-		const start = this.#index;
-
-		// Consume [
-		if (this.#text.charCodeAt(this.#index) !== LEFT_BRACKET) {
-			this.#index++;
-			this.#emit_text(this.#text[start]!, start);
-			return;
-		}
-		this.#index++;
-
-		// Emit link_text_open
-		this.#tokens.push({type: 'link_text_open', start, end: this.#index});
-
-		// Tokenize children until ]
-		while (this.#index < this.#text.length) {
-			if (this.#text.charCodeAt(this.#index) === RIGHT_BRACKET) break;
-			if (this.#is_at_paragraph_break()) break;
-			// Stop at ] and ) as delimiters
-			if (this.#text.charCodeAt(this.#index) === RIGHT_PAREN) break;
-			this.#tokenize_inline();
-		}
-
-		// Check for ]
-		if (this.#index >= this.#text.length || this.#text.charCodeAt(this.#index) !== RIGHT_BRACKET) {
-			// Revert - remove link_text_open and all children tokens added
-			this.#revert_tokens_from_link_open(start);
-			return;
-		}
-
-		const bracket_close_start = this.#index;
-		this.#index++; // consume ]
-		this.#tokens.push({
-			type: 'link_text_close',
-			start: bracket_close_start,
-			end: this.#index,
-		});
-
-		// Check for (
-		if (this.#index >= this.#text.length || this.#text.charCodeAt(this.#index) !== LEFT_PAREN) {
-			this.#revert_tokens_from_link_open(start);
-			return;
-		}
-		this.#index++; // consume (
-
-		// Find closing )
-		const close_paren = this.#text.indexOf(')', this.#index);
-		if (close_paren === -1) {
-			this.#revert_tokens_from_link_open(start);
-			return;
-		}
-
-		const reference = this.#text.slice(this.#index, close_paren);
-
-		// Validate reference
-		if (!reference.trim()) {
-			this.#revert_tokens_from_link_open(start);
-			return;
-		}
-
-		// Validate all characters
-		for (let i = 0; i < reference.length; i++) {
-			const char_code = reference.charCodeAt(i);
-			if (!is_valid_path_char(char_code)) {
-				this.#revert_tokens_from_link_open(start);
-				return;
-			}
-		}
-
-		// Reject unsafe protocols (javascript:, data:, etc.) — `is_valid_path_char`
-		// permits `:` so we need an explicit filter here.
-		if (!mdz_is_safe_reference(reference)) {
-			this.#revert_tokens_from_link_open(start);
-			return;
-		}
-
-		this.#index = close_paren + 1;
-
-		const link_type = mdz_is_url(reference) ? 'external' : 'internal';
-
-		this.#tokens.push({
-			type: 'link_ref',
-			reference,
-			link_type,
-			start: bracket_close_start + 1, // after ]
-			end: this.#index,
-		});
-	}
-
-	#revert_tokens_from_link_open(start: number): void {
-		// Find and remove link_text_open and all tokens after it
-		let open_idx = -1;
-		for (let i = this.#tokens.length - 1; i >= 0; i--) {
-			if (this.#tokens[i]!.type === 'link_text_open' && this.#tokens[i]!.start === start) {
-				open_idx = i;
-				break;
-			}
-		}
-		if (open_idx !== -1) {
-			this.#tokens.splice(open_idx);
-		}
-		this.#index = start + 1;
-		this.#emit_text('[', start);
-	}
-
-	#tokenize_tag(): void {
-		const start = this.#index;
-		this.#index++; // consume <
-
-		// Tag name must start with a letter
-		if (this.#index >= this.#text.length || !is_letter(this.#text.charCodeAt(this.#index))) {
-			this.#emit_text('<', start);
-			return;
-		}
-
-		// Collect tag name
-		const tag_name_start = this.#index;
-		while (
-			this.#index < this.#text.length &&
-			is_tag_name_char(this.#text.charCodeAt(this.#index))
-		) {
-			this.#index++;
-		}
-		const tag_name = this.#text.slice(tag_name_start, this.#index);
-
-		if (tag_name.length === 0) {
-			this.#emit_text('<', start);
-			return;
-		}
-
-		const first_char_code = tag_name.charCodeAt(0);
-		const is_component = first_char_code >= A_UPPER && first_char_code <= Z_UPPER;
-
-		// Skip whitespace
-		while (this.#index < this.#text.length && this.#text.charCodeAt(this.#index) === SPACE) {
-			this.#index++;
-		}
-
-		// Check for self-closing />
-		if (
-			this.#index + 1 < this.#text.length &&
-			this.#text.charCodeAt(this.#index) === SLASH &&
-			this.#text.charCodeAt(this.#index + 1) === RIGHT_ANGLE
-		) {
-			this.#index += 2;
-			this.#tokens.push({
-				type: 'tag_self_close',
-				name: tag_name,
-				is_component,
-				start,
-				end: this.#index,
-			});
-			return;
-		}
-
-		// Check for >
-		if (this.#index >= this.#text.length || this.#text.charCodeAt(this.#index) !== RIGHT_ANGLE) {
-			this.#index = start + 1;
-			this.#emit_text('<', start);
-			return;
-		}
-		this.#index++; // consume >
-
-		// Check for closing tag existence before committing —
-		// must exist within search boundary and before any paragraph break
-		const closing_tag = `</${tag_name}>`;
-		const search_limit = Math.min(this.#max_search_index, this.#text.length);
-		const closing_tag_pos = this.#text.indexOf(closing_tag, this.#index);
-		if (closing_tag_pos === -1 || closing_tag_pos >= search_limit) {
-			this.#index = start + 1;
-			this.#emit_text('<', start);
-			return;
-		}
-		if (this.#has_paragraph_break_between(this.#index, closing_tag_pos)) {
-			this.#index = start + 1;
-			this.#emit_text('<', start);
-			return;
-		}
-
-		// Emit tag_open
-		this.#tokens.push({type: 'tag_open', name: tag_name, is_component, start, end: this.#index});
-
-		// Tokenize children until closing tag
-		while (this.#index < this.#text.length) {
-			if (this.#match(closing_tag)) {
-				const close_start = this.#index;
-				this.#index += closing_tag.length;
-				this.#tokens.push({
-					type: 'tag_close',
-					name: tag_name,
-					start: close_start,
-					end: this.#index,
-				});
-				return;
-			}
-			this.#tokenize_inline();
-		}
-
-		// Shouldn't reach here since we verified closing tag exists
-		// But if we do, the tag_open token stays and parser will handle it
-	}
-
-	#tokenize_text(): void {
-		const start = this.#index;
-
-		// Check for URL or internal path at current position
-		if (this.#is_at_url()) {
-			this.#tokenize_auto_link_url();
-			return;
-		}
-		if (is_at_absolute_path(this.#text, this.#index)) {
-			this.#tokenize_auto_link_internal();
-			return;
-		}
-		if (is_at_relative_path(this.#text, this.#index)) {
-			this.#tokenize_auto_link_internal();
-			return;
-		}
-
-		while (this.#index < this.#text.length) {
-			const char_code = this.#text.charCodeAt(this.#index);
-
-			// Stop at special characters
-			if (
-				char_code === BACKTICK ||
-				char_code === ASTERISK ||
-				char_code === UNDERSCORE ||
-				char_code === TILDE ||
-				char_code === LEFT_BRACKET ||
-				char_code === RIGHT_BRACKET ||
-				char_code === RIGHT_PAREN ||
-				char_code === LEFT_ANGLE
-			) {
-				break;
-			}
-
-			// Check for paragraph break
-			if (this.#is_at_paragraph_break()) break;
-
-			// When next line could start a block element, consume the newline and stop
-			if (char_code === NEWLINE) {
-				const next_i = this.#index + 1;
-				if (next_i < this.#text.length) {
-					const next_char = this.#text.charCodeAt(next_i);
-					if (next_char === HASH || next_char === HYPHEN || next_char === BACKTICK) {
-						this.#index++; // consume the newline
-						break;
-					}
-				}
-			}
-
-			// Check for URL or internal path mid-text (char code guard avoids startsWith on every char)
-			if (
-				((char_code === 104 /* h */ || char_code === 72) /* H */ && this.#is_at_url()) ||
-				(char_code === SLASH && is_at_absolute_path(this.#text, this.#index)) ||
-				(char_code === PERIOD && is_at_relative_path(this.#text, this.#index))
-			) {
-				break;
-			}
-
-			this.#index++;
-		}
-
-		// Ensure we always consume at least one character
-		if (this.#index === start && this.#index < this.#text.length) {
-			this.#index++;
-		}
-
-		const content = this.#text.slice(start, this.#index);
-		this.#emit_text(content, start);
-	}
-
-	// -- Auto-link tokenizers --
-
-	#tokenize_auto_link_url(): void {
-		const start = this.#index;
-
-		// Consume protocol (case-insensitive match; original casing preserved in reference)
-		this.#index += match_url_prefix_case_insensitive(this.#text, this.#index);
-
-		// Collect URL characters
-		while (this.#index < this.#text.length) {
-			const char_code = this.#text.charCodeAt(this.#index);
-			if (char_code === SPACE || char_code === NEWLINE || !is_valid_path_char(char_code)) {
-				break;
-			}
-			this.#index++;
-		}
-
-		let reference = this.#text.slice(start, this.#index);
-		reference = trim_trailing_punctuation(reference);
-		this.#index = start + reference.length;
-
-		this.#tokens.push({
-			type: 'autolink',
-			reference,
-			link_type: 'external',
-			start,
-			end: this.#index,
-		});
-	}
-
-	#tokenize_auto_link_internal(): void {
-		const start = this.#index;
-
-		// Collect path characters
-		while (this.#index < this.#text.length) {
-			const char_code = this.#text.charCodeAt(this.#index);
-			if (char_code === SPACE || char_code === NEWLINE || !is_valid_path_char(char_code)) {
-				break;
-			}
-			this.#index++;
-		}
-
-		let reference = this.#text.slice(start, this.#index);
-		reference = trim_trailing_punctuation(reference);
-		this.#index = start + reference.length;
-
-		this.#tokens.push({
-			type: 'autolink',
-			reference,
-			link_type: 'internal',
-			start,
-			end: this.#index,
-		});
-	}
-
-	// -- Helper methods --
-
-	#emit_text(content: string, start: number): void {
-		this.#tokens.push({type: 'text', content, start, end: start + content.length});
-	}
-
-	#has_paragraph_break_between(from: number, to: number): boolean {
-		for (let i = from; i < to - 1; i++) {
-			if (this.#text.charCodeAt(i) === NEWLINE && this.#text.charCodeAt(i + 1) === NEWLINE) {
-				return true;
-			}
-		}
-		return false;
-	}
-
-	#is_at_url(): boolean {
-		// Scheme matching is case-insensitive (RFC 3986).
-		const prefix_len = match_url_prefix_case_insensitive(this.#text, this.#index);
-		if (prefix_len === 0) return false;
-		if (this.#index + prefix_len >= this.#text.length) return false;
-		const next_char = this.#text.charCodeAt(this.#index + prefix_len);
-		return next_char !== SPACE && next_char !== NEWLINE;
-	}
-
-	#is_at_word_boundary(index: number, check_before: boolean, check_after: boolean): boolean {
-		if (check_before && index > 0) {
-			const prev = this.#text.charCodeAt(index - 1);
-			if (is_word_char(prev)) return false;
-		}
-		if (check_after && index < this.#text.length) {
-			const next = this.#text.charCodeAt(index);
-			if (is_word_char(next)) return false;
-		}
-		return true;
-	}
-}